mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
refactor(model-cost): derive the provider model sets from the live map [12/12]
The 121 provider sets and models_by_provider are read-only views over the catalog's index, grouped by raw litellm_provider, so a direct write, reload or registration shows on the next read and a moved or dropped key leaves its old set. The populate chain and the models_by_provider builder are gone, and every provider in the map is listed Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
9f337ea9b5
commit
f799435484
26 changed files with 739 additions and 812 deletions
|
|
@ -18,7 +18,7 @@ use litellm_model_catalog::{BUNDLED_REGISTRY, Catalog, Keys, Layers, Located};
|
|||
use litellm_model_catalog_refresh::{Source, git_blob_id};
|
||||
use pyo3::exceptions::PyTypeError;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::{PyDict, PyModule};
|
||||
use pyo3::types::{PyDict, PyFrozenSet, PyModule};
|
||||
use serde::Serialize;
|
||||
|
||||
pub(crate) use fetch::{Bundled, File, Provenance, Remote};
|
||||
|
|
@ -333,6 +333,30 @@ impl CatalogHandle {
|
|||
Ok(keys.provider_keys(provider))
|
||||
}
|
||||
|
||||
/// The keys whose live row names this raw `litellm_provider`, empty when none (decision 18)
|
||||
fn models_for<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
provider: &str,
|
||||
) -> PyResult<Bound<'py, PyFrozenSet>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
PyFrozenSet::new(py, keys.provider_keys(provider))
|
||||
}
|
||||
|
||||
/// Every raw `litellm_provider` string a live row names, sorted
|
||||
fn providers(&self, py: Python<'_>) -> PyResult<Vec<String>> {
|
||||
let dict = self.bound_dict(py)?;
|
||||
let keys = resolve::LiveKeys {
|
||||
dict: &dict,
|
||||
index: &self.index,
|
||||
};
|
||||
Ok(keys.providers())
|
||||
}
|
||||
|
||||
/// main's 133-field copy of a row Python built, a generalization match or a fill-missing merge
|
||||
#[staticmethod]
|
||||
#[pyo3(signature = (key, row, provider = None))]
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::{PyDict, PyString};
|
||||
|
||||
/// The one name index of decision 18. A normalized name to the actual key, and the keys each raw `litellm_provider`
|
||||
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale
|
||||
/// names. Never a row, since rows are read live, so only adding or removing keys can make it stale. It holds the dict it
|
||||
/// indexed, so a freed dict's id can never be taken for it by a new dict of the same length
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct NameIndex {
|
||||
dict_id: usize,
|
||||
dict: Option<Py<PyDict>>,
|
||||
len: usize,
|
||||
by_lower: HashMap<String, String>,
|
||||
by_provider: HashMap<String, Vec<String>>,
|
||||
|
|
@ -32,7 +33,7 @@ impl NameIndex {
|
|||
by_lower.insert(key.to_lowercase(), key);
|
||||
}
|
||||
Self {
|
||||
dict_id: dict.as_ptr() as usize,
|
||||
dict: Some(dict.clone().unbind()),
|
||||
len: dict.len(),
|
||||
by_lower,
|
||||
by_provider,
|
||||
|
|
@ -40,13 +41,23 @@ impl NameIndex {
|
|||
}
|
||||
|
||||
pub(crate) fn is_current(&self, dict: &Bound<'_, PyDict>) -> bool {
|
||||
self.dict_id == dict.as_ptr() as usize && self.len == dict.len()
|
||||
self.dict
|
||||
.as_ref()
|
||||
.is_some_and(|indexed| indexed.as_ptr() == dict.as_ptr())
|
||||
&& self.len == dict.len()
|
||||
}
|
||||
|
||||
pub(crate) fn by_lower(&self, name: &str) -> Option<&str> {
|
||||
self.by_lower.get(&name.to_lowercase()).map(String::as_str)
|
||||
}
|
||||
|
||||
/// Every raw `litellm_provider` string some live row names, sorted
|
||||
pub(crate) fn providers(&self) -> Vec<String> {
|
||||
let mut providers: Vec<String> = self.by_provider.keys().cloned().collect();
|
||||
providers.sort();
|
||||
providers
|
||||
}
|
||||
|
||||
pub(crate) fn provider_keys(&self, provider: &str) -> &[String] {
|
||||
self.by_provider.get(provider).map_or(&[], Vec::as_slice)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,6 +106,10 @@ impl LiveKeys<'_, '_> {
|
|||
Some(NameIndex::build(self.dict));
|
||||
}
|
||||
|
||||
pub(crate) fn providers(&self) -> Vec<String> {
|
||||
self.indexed(NameIndex::providers)
|
||||
}
|
||||
|
||||
pub(crate) fn provider_keys(&self, provider: &str) -> Vec<String> {
|
||||
self.indexed(|index| index.provider_keys(provider).to_vec())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ _apply_process_defaults()
|
|||
### INIT VARIABLES #########################
|
||||
import threading
|
||||
import os
|
||||
import datetime
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import (
|
||||
|
|
@ -39,7 +38,6 @@ from litellm._logging import (
|
|||
turn_on_json,
|
||||
log_level,
|
||||
)
|
||||
import re
|
||||
from litellm.constants import (
|
||||
DEFAULT_BATCH_SIZE,
|
||||
DEFAULT_FLUSH_INTERVAL_SECONDS,
|
||||
|
|
@ -61,9 +59,9 @@ from litellm.constants import (
|
|||
clarifai_models,
|
||||
huggingface_models,
|
||||
modelscope_models,
|
||||
empower_models,
|
||||
empower_models as empower_models,
|
||||
together_ai_models,
|
||||
WANDB_MODELS,
|
||||
WANDB_MODELS as WANDB_MODELS,
|
||||
REPEATED_STREAMING_CHUNK_LIMIT,
|
||||
request_timeout,
|
||||
request_timeout_explicitly_set as request_timeout_explicitly_set,
|
||||
|
|
@ -73,7 +71,7 @@ from litellm.constants import (
|
|||
known_tokenizer_config,
|
||||
BEDROCK_INVOKE_PROVIDERS_LITERAL,
|
||||
BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
|
||||
BEDROCK_CONVERSE_MODELS,
|
||||
BEDROCK_CONVERSE_MODELS as BEDROCK_CONVERSE_MODELS,
|
||||
DEFAULT_MAX_TOKENS,
|
||||
DEFAULT_SOFT_BUDGET,
|
||||
DEFAULT_ALLOWED_FAILS,
|
||||
|
|
@ -548,6 +546,14 @@ _key_management_system: Optional["KeyManagementSystem"] = None
|
|||
#### PII MASKING ####
|
||||
output_parse_pii: bool = False
|
||||
#############################################
|
||||
from litellm._catalog.provider_sets import (
|
||||
LEGACY_NAMES as _LEGACY_PROVIDER_SETS,
|
||||
ModelsByProvider,
|
||||
_is_past_deprecation_date as _is_past_deprecation_date, # pyright: ignore[reportPrivateUsage] # main's name
|
||||
is_bedrock_pricing_only_model as is_bedrock_pricing_only_model,
|
||||
is_openai_finetune_model as is_openai_finetune_model,
|
||||
legacy_set as _provider_set,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
get_model_cost_map as get_model_cost_map,
|
||||
load_bundled_model_cost_map_at_import,
|
||||
|
|
@ -594,455 +600,19 @@ config_path = None
|
|||
vertex_ai_safety_settings: Optional[dict] = None
|
||||
|
||||
####### COMPLETION MODELS ###################
|
||||
from typing import Set
|
||||
|
||||
open_ai_chat_completion_models: Set = set()
|
||||
open_ai_text_completion_models: Set = set()
|
||||
cohere_models: Set = set()
|
||||
cohere_chat_models: Set = set()
|
||||
mistral_chat_models: Set = set()
|
||||
text_completion_codestral_models: Set = set()
|
||||
text_completion_inception_models: Set = set()
|
||||
anthropic_models: Set = set()
|
||||
openrouter_models: Set = set()
|
||||
datarobot_models: Set = set()
|
||||
vertex_language_models: Set = set()
|
||||
vertex_vision_models: Set = set()
|
||||
vertex_chat_models: Set = set()
|
||||
vertex_code_chat_models: Set = set()
|
||||
vertex_ai_image_models: Set = set()
|
||||
vertex_ai_video_models: Set = set()
|
||||
vertex_text_models: Set = set()
|
||||
vertex_code_text_models: Set = set()
|
||||
vertex_embedding_models: Set = set()
|
||||
vertex_anthropic_models: Set = set()
|
||||
vertex_llama3_models: Set = set()
|
||||
vertex_deepseek_models: Set = set()
|
||||
vertex_ai_ai21_models: Set = set()
|
||||
vertex_mistral_models: Set = set()
|
||||
vertex_openai_models: Set = set()
|
||||
vertex_minimax_models: Set = set()
|
||||
vertex_moonshot_models: Set = set()
|
||||
vertex_zai_models: Set = set()
|
||||
ai21_models: Set = set()
|
||||
ai21_chat_models: Set = set()
|
||||
nlp_cloud_models: Set = set()
|
||||
aleph_alpha_models: Set = set()
|
||||
bedrock_models: Set = set()
|
||||
bedrock_converse_models: Set = set(BEDROCK_CONVERSE_MODELS)
|
||||
fal_ai_models: Set = set()
|
||||
fireworks_ai_models: Set = set()
|
||||
baseten_models: Set[str] = set()
|
||||
fireworks_ai_embedding_models: Set = set()
|
||||
deepinfra_models: Set = set()
|
||||
perplexity_models: Set = set()
|
||||
watsonx_models: Set = set()
|
||||
gemini_models: Set = set()
|
||||
xai_models: Set = set()
|
||||
zai_models: Set = set()
|
||||
deepseek_models: Set = set()
|
||||
tencent_models: Set = set()
|
||||
runwayml_models: Set = set()
|
||||
azure_ai_models: Set = set()
|
||||
jina_ai_models: Set = set()
|
||||
voyage_models: Set = set()
|
||||
infinity_models: Set = set()
|
||||
heroku_models: Set = set()
|
||||
databricks_models: Set = set()
|
||||
cloudflare_models: Set = set()
|
||||
codestral_models: Set = set()
|
||||
friendliai_models: Set = set()
|
||||
featherless_ai_models: Set = set()
|
||||
palm_models: Set = set()
|
||||
groq_models: Set = set()
|
||||
azure_models: Set = set()
|
||||
azure_anthropic_models: Set = set()
|
||||
azure_text_models: Set = set()
|
||||
anyscale_models: Set = set()
|
||||
cerebras_models: Set = set()
|
||||
nadir_models: Set = set()
|
||||
galadriel_models: Set = set()
|
||||
nvidia_nim_models: Set = set()
|
||||
nvidia_riva_models: Set = set()
|
||||
soniox_models: Set = set()
|
||||
sambanova_models: Set = set()
|
||||
scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references.
|
||||
sambanova_embedding_models: Set = set()
|
||||
novita_models: Set = set()
|
||||
assemblyai_models: Set = set()
|
||||
snowflake_models: Set = set()
|
||||
gradient_ai_models: Set = set()
|
||||
llama_models: Set = set()
|
||||
nscale_models: Set = set()
|
||||
nebius_models: Set = set()
|
||||
nebius_embedding_models: Set = set()
|
||||
aiml_models: Set = set()
|
||||
deepgram_models: Set = set()
|
||||
elevenlabs_models: Set = set()
|
||||
dashscope_models: Set = set()
|
||||
qwencloud_models: Set = set()
|
||||
qwen_ai_platform_models: Set = set()
|
||||
moonshot_models: Set = set()
|
||||
publicai_models: Set = set()
|
||||
darkbloom_models: Set = set()
|
||||
openinfer_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
lambda_ai_models: Set = set()
|
||||
inception_models: Set = set()
|
||||
hyperbolic_models: Set = set()
|
||||
black_forest_labs_models: Set = set()
|
||||
recraft_models: Set = set()
|
||||
cometapi_models: Set = set()
|
||||
oci_models: Set = set()
|
||||
vercel_ai_gateway_models: Set = set()
|
||||
edenai_models: Set = set()
|
||||
volcengine_models: Set = set()
|
||||
wandb_models: Set = set(WANDB_MODELS)
|
||||
ovhcloud_models: Set = set()
|
||||
ovhcloud_embedding_models: Set = set()
|
||||
lemonade_models: Set = set()
|
||||
docker_model_runner_models: Set = set()
|
||||
amazon_nova_models: Set = set()
|
||||
stability_models: Set = set()
|
||||
github_copilot_models: Set = set()
|
||||
microsoft_365_copilot_models: Set[str] = set()
|
||||
chatgpt_models: Set = set()
|
||||
minimax_models: Set = set()
|
||||
aws_polly_models: Set = set()
|
||||
transcribe_models: Set = set()
|
||||
gigachat_models: Set = set()
|
||||
llamagate_models: Set = set()
|
||||
coralbricks_models: Set[str] = set()
|
||||
reducto_models: Set = set()
|
||||
bedrock_mantle_models: Set = set()
|
||||
|
||||
|
||||
def is_bedrock_pricing_only_model(key: str) -> bool:
|
||||
"""
|
||||
Excludes keys with the pattern 'bedrock/<region>/<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
|
||||
|
||||
Args:
|
||||
key (str): A key to filter.
|
||||
|
||||
Returns:
|
||||
bool: True if the key matches the Bedrock pattern, False otherwise.
|
||||
"""
|
||||
# Regex to match 'bedrock/<region>/<model>'
|
||||
bedrock_pattern: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$")
|
||||
|
||||
if "month-commitment" in key:
|
||||
return True
|
||||
|
||||
is_match: Final = bedrock_pattern.match(key)
|
||||
return is_match is not None
|
||||
|
||||
|
||||
def is_openai_finetune_model(key: str) -> bool:
|
||||
"""
|
||||
Excludes model cost keys with the pattern 'ft:<model>'. These are in the model_prices_and_context_window.json file for pricing purposes only.
|
||||
|
||||
Args:
|
||||
key (str): A key to filter.
|
||||
|
||||
Returns:
|
||||
bool: True if the key matches the OpenAI finetune pattern, False otherwise.
|
||||
"""
|
||||
return key.startswith("ft:") and not key.count(":") > 1
|
||||
|
||||
|
||||
def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
|
||||
deprecation_date: Final = value.get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None:
|
||||
for key, value in tuple(model_cost_map.items()):
|
||||
if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key):
|
||||
open_ai_chat_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-openai":
|
||||
open_ai_text_completion_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure_text":
|
||||
azure_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere":
|
||||
cohere_models.add(key)
|
||||
elif value.get("litellm_provider") == "cohere_chat":
|
||||
cohere_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "mistral":
|
||||
mistral_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "anthropic":
|
||||
anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "empower":
|
||||
empower_models.add(key)
|
||||
elif value.get("litellm_provider") == "openrouter":
|
||||
openrouter_models.add(key)
|
||||
elif value.get("litellm_provider") == "vercel_ai_gateway":
|
||||
vercel_ai_gateway_models.add(key)
|
||||
elif value.get("litellm_provider") == "edenai":
|
||||
edenai_models.add(key)
|
||||
elif value.get("litellm_provider") == "datarobot":
|
||||
datarobot_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-text-models":
|
||||
vertex_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-text-models":
|
||||
vertex_code_text_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-language-models":
|
||||
vertex_language_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-vision-models":
|
||||
vertex_vision_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-chat-models":
|
||||
vertex_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-code-chat-models":
|
||||
vertex_code_chat_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-embedding-models":
|
||||
vertex_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-anthropic_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-llama_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_llama3_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-deepseek_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_deepseek_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-mistral_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_mistral_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-ai21_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_ai_ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-image-models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_ai_image_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-video-models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_ai_video_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-openai_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_openai_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-minimax_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_minimax_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-moonshot_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_moonshot_models.add(key)
|
||||
elif value.get("litellm_provider") == "vertex_ai-zai_models":
|
||||
key = key.replace("vertex_ai/", "")
|
||||
vertex_zai_models.add(key)
|
||||
elif value.get("litellm_provider") == "ai21":
|
||||
if value.get("mode") == "chat":
|
||||
ai21_chat_models.add(key)
|
||||
else:
|
||||
ai21_models.add(key)
|
||||
elif value.get("litellm_provider") == "nlp_cloud":
|
||||
nlp_cloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "aleph_alpha":
|
||||
aleph_alpha_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail":
|
||||
pass
|
||||
elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key):
|
||||
bedrock_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_converse":
|
||||
bedrock_converse_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepinfra":
|
||||
deepinfra_models.add(key)
|
||||
elif value.get("litellm_provider") == "perplexity":
|
||||
perplexity_models.add(key)
|
||||
elif value.get("litellm_provider") == "watsonx":
|
||||
watsonx_models.add(key)
|
||||
elif value.get("litellm_provider") == "gemini":
|
||||
gemini_models.add(key)
|
||||
elif value.get("litellm_provider") == "fireworks_ai":
|
||||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "baseten":
|
||||
if _is_past_deprecation_date(value):
|
||||
baseten_models.discard(key)
|
||||
else:
|
||||
baseten_models.add(key)
|
||||
elif value.get("litellm_provider") == "fireworks_ai-embedding-models":
|
||||
# ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
|
||||
if "-to-" not in key:
|
||||
fireworks_ai_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-codestral":
|
||||
text_completion_codestral_models.add(key)
|
||||
elif value.get("litellm_provider") == "text-completion-inception":
|
||||
text_completion_inception_models.add(key)
|
||||
elif value.get("litellm_provider") == "xai":
|
||||
xai_models.add(key)
|
||||
elif value.get("litellm_provider") == "zai":
|
||||
zai_models.add(key)
|
||||
elif value.get("litellm_provider") == "fal_ai":
|
||||
fal_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepseek":
|
||||
deepseek_models.add(key)
|
||||
elif value.get("litellm_provider") == "tencent":
|
||||
tencent_models.add(key)
|
||||
elif value.get("litellm_provider") == "runwayml":
|
||||
runwayml_models.add(key)
|
||||
elif value.get("litellm_provider") == "meta_llama":
|
||||
llama_models.add(key)
|
||||
elif value.get("litellm_provider") == "nscale":
|
||||
nscale_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure_ai":
|
||||
azure_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "voyage":
|
||||
voyage_models.add(key)
|
||||
elif value.get("litellm_provider") == "infinity":
|
||||
infinity_models.add(key)
|
||||
elif value.get("litellm_provider") == "databricks":
|
||||
databricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "cloudflare":
|
||||
cloudflare_models.add(key)
|
||||
elif value.get("litellm_provider") == "codestral":
|
||||
codestral_models.add(key)
|
||||
elif value.get("litellm_provider") == "friendliai":
|
||||
friendliai_models.add(key)
|
||||
elif value.get("litellm_provider") == "palm":
|
||||
palm_models.add(key)
|
||||
elif value.get("litellm_provider") == "groq":
|
||||
groq_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure":
|
||||
azure_models.add(key)
|
||||
elif value.get("litellm_provider") == "azure_anthropic":
|
||||
azure_anthropic_models.add(key)
|
||||
elif value.get("litellm_provider") == "anyscale":
|
||||
anyscale_models.add(key)
|
||||
elif value.get("litellm_provider") == "cerebras":
|
||||
cerebras_models.add(key)
|
||||
elif value.get("litellm_provider") == "nadir":
|
||||
nadir_models.add(key)
|
||||
elif value.get("litellm_provider") == "galadriel":
|
||||
galadriel_models.add(key)
|
||||
elif value.get("litellm_provider") == "nvidia_nim":
|
||||
nvidia_nim_models.add(key)
|
||||
elif value.get("litellm_provider") == "nvidia_riva":
|
||||
nvidia_riva_models.add(key)
|
||||
elif value.get("litellm_provider") == "soniox":
|
||||
soniox_models.add(key)
|
||||
elif value.get("litellm_provider") == "sambanova":
|
||||
sambanova_models.add(key)
|
||||
elif value.get("litellm_provider") == "scaledown":
|
||||
scaledown_models.add(key)
|
||||
elif value.get("litellm_provider") == "sambanova-embedding-models":
|
||||
sambanova_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "novita":
|
||||
novita_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-chat-models":
|
||||
nebius_models.add(key)
|
||||
elif value.get("litellm_provider") == "nebius-embedding-models":
|
||||
nebius_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "aiml":
|
||||
aiml_models.add(key)
|
||||
elif value.get("litellm_provider") == "assemblyai":
|
||||
assemblyai_models.add(key)
|
||||
elif value.get("litellm_provider") == "jina_ai":
|
||||
jina_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "snowflake":
|
||||
snowflake_models.add(key)
|
||||
elif value.get("litellm_provider") == "gradient_ai":
|
||||
gradient_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "featherless_ai":
|
||||
featherless_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "deepgram":
|
||||
deepgram_models.add(key)
|
||||
elif value.get("litellm_provider") == "elevenlabs":
|
||||
elevenlabs_models.add(key)
|
||||
elif value.get("litellm_provider") == "heroku":
|
||||
heroku_models.add(key)
|
||||
elif value.get("litellm_provider") == "dashscope":
|
||||
dashscope_models.add(key)
|
||||
elif value.get("litellm_provider") == "qwencloud":
|
||||
qwencloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "qwen_ai_platform":
|
||||
qwen_ai_platform_models.add(key)
|
||||
elif value.get("litellm_provider") == "modelscope":
|
||||
modelscope_models.add(key)
|
||||
elif value.get("litellm_provider") == "moonshot":
|
||||
moonshot_models.add(key)
|
||||
elif value.get("litellm_provider") == "publicai":
|
||||
publicai_models.add(key)
|
||||
elif value.get("litellm_provider") == "darkbloom":
|
||||
darkbloom_models.add(key)
|
||||
elif value.get("litellm_provider") == "openinfer":
|
||||
openinfer_models.add(key)
|
||||
elif value.get("litellm_provider") == "v0":
|
||||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "opencode_go":
|
||||
opencode_go_models.add(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "inception":
|
||||
inception_models.add(key)
|
||||
elif value.get("litellm_provider") == "hyperbolic":
|
||||
hyperbolic_models.add(key)
|
||||
elif value.get("litellm_provider") == "black_forest_labs":
|
||||
black_forest_labs_models.add(key)
|
||||
elif value.get("litellm_provider") == "recraft":
|
||||
recraft_models.add(key)
|
||||
elif value.get("litellm_provider") == "cometapi":
|
||||
cometapi_models.add(key)
|
||||
elif value.get("litellm_provider") == "oci":
|
||||
oci_models.add(key)
|
||||
elif value.get("litellm_provider") == "volcengine":
|
||||
volcengine_models.add(key)
|
||||
elif value.get("litellm_provider") == "wandb":
|
||||
wandb_models.add(key)
|
||||
elif value.get("litellm_provider") == "ovhcloud":
|
||||
ovhcloud_models.add(key)
|
||||
elif value.get("litellm_provider") == "ovhcloud-embedding-models":
|
||||
ovhcloud_embedding_models.add(key)
|
||||
elif value.get("litellm_provider") == "lemonade":
|
||||
lemonade_models.add(key)
|
||||
elif value.get("litellm_provider") == "docker_model_runner":
|
||||
docker_model_runner_models.add(key)
|
||||
elif value.get("litellm_provider") == "amazon_nova":
|
||||
amazon_nova_models.add(key)
|
||||
elif value.get("litellm_provider") == "stability":
|
||||
stability_models.add(key)
|
||||
elif value.get("litellm_provider") == "github_copilot":
|
||||
github_copilot_models.add(key)
|
||||
elif value.get("litellm_provider") == "microsoft_365_copilot":
|
||||
microsoft_365_copilot_models.add(key)
|
||||
elif value.get("litellm_provider") == "chatgpt":
|
||||
chatgpt_models.add(key)
|
||||
elif value.get("litellm_provider") == "minimax":
|
||||
minimax_models.add(key)
|
||||
elif value.get("litellm_provider") == "aws_polly":
|
||||
aws_polly_models.add(key)
|
||||
elif value.get("litellm_provider") == "transcribe":
|
||||
transcribe_models.add(key)
|
||||
elif value.get("litellm_provider") == "gigachat":
|
||||
gigachat_models.add(key)
|
||||
elif value.get("litellm_provider") == "llamagate":
|
||||
llamagate_models.add(key)
|
||||
elif value.get("litellm_provider") == "coralbricks":
|
||||
coralbricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "reducto":
|
||||
reducto_models.add(key)
|
||||
elif value.get("litellm_provider") == "bedrock_mantle":
|
||||
bedrock_mantle_models.add(key)
|
||||
|
||||
|
||||
def add_known_models(model_cost_map: Optional[Dict] = None):
|
||||
"""Fold `model_cost_map` (defaults to `litellm.model_cost`) into the per-provider model sets,
|
||||
then refresh `models_by_provider` from those sets so the additions reach wildcard expansion.
|
||||
The refresh updates the dict in place, so references captured before a reload stay live.
|
||||
"""
|
||||
_populate_provider_model_sets(model_cost_map if model_cost_map is not None else model_cost)
|
||||
models_by_provider.update(_build_models_by_provider())
|
||||
|
||||
|
||||
_populate_provider_model_sets(model_cost)
|
||||
|
||||
# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
|
||||
|
||||
# this is maintained for Exception Mapping
|
||||
|
|
@ -1073,97 +643,97 @@ ollama_models = ["llama2"]
|
|||
maritalk_models = ["maritalk"]
|
||||
|
||||
model_list = list(
|
||||
open_ai_chat_completion_models
|
||||
| open_ai_text_completion_models
|
||||
| cohere_models
|
||||
| cohere_chat_models
|
||||
| anthropic_models
|
||||
_provider_set("open_ai_chat_completion_models")
|
||||
| _provider_set("open_ai_text_completion_models")
|
||||
| _provider_set("cohere_models")
|
||||
| _provider_set("cohere_chat_models")
|
||||
| _provider_set("anthropic_models")
|
||||
| set(replicate_models)
|
||||
| openrouter_models
|
||||
| datarobot_models
|
||||
| _provider_set("openrouter_models")
|
||||
| _provider_set("datarobot_models")
|
||||
| set(huggingface_models)
|
||||
| vertex_chat_models
|
||||
| vertex_text_models
|
||||
| ai21_models
|
||||
| ai21_chat_models
|
||||
| _provider_set("vertex_chat_models")
|
||||
| _provider_set("vertex_text_models")
|
||||
| _provider_set("ai21_models")
|
||||
| _provider_set("ai21_chat_models")
|
||||
| set(together_ai_models)
|
||||
| set(baseten_models)
|
||||
| aleph_alpha_models
|
||||
| nlp_cloud_models
|
||||
| set(_provider_set("baseten_models"))
|
||||
| _provider_set("aleph_alpha_models")
|
||||
| _provider_set("nlp_cloud_models")
|
||||
| set(ollama_models)
|
||||
| bedrock_models
|
||||
| deepinfra_models
|
||||
| perplexity_models
|
||||
| _provider_set("bedrock_models")
|
||||
| _provider_set("deepinfra_models")
|
||||
| _provider_set("perplexity_models")
|
||||
| set(maritalk_models)
|
||||
| runwayml_models
|
||||
| vertex_language_models
|
||||
| watsonx_models
|
||||
| gemini_models
|
||||
| text_completion_codestral_models
|
||||
| text_completion_inception_models
|
||||
| xai_models
|
||||
| zai_models
|
||||
| fal_ai_models
|
||||
| deepseek_models
|
||||
| _provider_set("runwayml_models")
|
||||
| _provider_set("vertex_language_models")
|
||||
| _provider_set("watsonx_models")
|
||||
| _provider_set("gemini_models")
|
||||
| _provider_set("text_completion_codestral_models")
|
||||
| _provider_set("text_completion_inception_models")
|
||||
| _provider_set("xai_models")
|
||||
| _provider_set("zai_models")
|
||||
| _provider_set("fal_ai_models")
|
||||
| _provider_set("deepseek_models")
|
||||
| modelscope_models
|
||||
| azure_ai_models
|
||||
| voyage_models
|
||||
| infinity_models
|
||||
| databricks_models
|
||||
| cloudflare_models
|
||||
| codestral_models
|
||||
| friendliai_models
|
||||
| palm_models
|
||||
| groq_models
|
||||
| azure_models
|
||||
| azure_anthropic_models
|
||||
| anyscale_models
|
||||
| cerebras_models
|
||||
| nadir_models
|
||||
| galadriel_models
|
||||
| nvidia_nim_models
|
||||
| nvidia_riva_models
|
||||
| soniox_models
|
||||
| sambanova_models
|
||||
| scaledown_models
|
||||
| azure_text_models
|
||||
| novita_models
|
||||
| assemblyai_models
|
||||
| jina_ai_models
|
||||
| snowflake_models
|
||||
| gradient_ai_models
|
||||
| llama_models
|
||||
| featherless_ai_models
|
||||
| nscale_models
|
||||
| deepgram_models
|
||||
| elevenlabs_models
|
||||
| dashscope_models
|
||||
| qwencloud_models
|
||||
| qwen_ai_platform_models
|
||||
| moonshot_models
|
||||
| publicai_models
|
||||
| darkbloom_models
|
||||
| openinfer_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
| lambda_ai_models
|
||||
| inception_models
|
||||
| black_forest_labs_models
|
||||
| recraft_models
|
||||
| cometapi_models
|
||||
| oci_models
|
||||
| heroku_models
|
||||
| vercel_ai_gateway_models
|
||||
| edenai_models
|
||||
| volcengine_models
|
||||
| wandb_models
|
||||
| ovhcloud_models
|
||||
| lemonade_models
|
||||
| docker_model_runner_models
|
||||
| reducto_models
|
||||
| bedrock_mantle_models
|
||||
| coralbricks_models
|
||||
| _provider_set("azure_ai_models")
|
||||
| _provider_set("voyage_models")
|
||||
| _provider_set("infinity_models")
|
||||
| _provider_set("databricks_models")
|
||||
| _provider_set("cloudflare_models")
|
||||
| _provider_set("codestral_models")
|
||||
| _provider_set("friendliai_models")
|
||||
| _provider_set("palm_models")
|
||||
| _provider_set("groq_models")
|
||||
| _provider_set("azure_models")
|
||||
| _provider_set("azure_anthropic_models")
|
||||
| _provider_set("anyscale_models")
|
||||
| _provider_set("cerebras_models")
|
||||
| _provider_set("nadir_models")
|
||||
| _provider_set("galadriel_models")
|
||||
| _provider_set("nvidia_nim_models")
|
||||
| _provider_set("nvidia_riva_models")
|
||||
| _provider_set("soniox_models")
|
||||
| _provider_set("sambanova_models")
|
||||
| _provider_set("scaledown_models")
|
||||
| _provider_set("azure_text_models")
|
||||
| _provider_set("novita_models")
|
||||
| _provider_set("assemblyai_models")
|
||||
| _provider_set("jina_ai_models")
|
||||
| _provider_set("snowflake_models")
|
||||
| _provider_set("gradient_ai_models")
|
||||
| _provider_set("llama_models")
|
||||
| _provider_set("featherless_ai_models")
|
||||
| _provider_set("nscale_models")
|
||||
| _provider_set("deepgram_models")
|
||||
| _provider_set("elevenlabs_models")
|
||||
| _provider_set("dashscope_models")
|
||||
| _provider_set("qwencloud_models")
|
||||
| _provider_set("qwen_ai_platform_models")
|
||||
| _provider_set("moonshot_models")
|
||||
| _provider_set("publicai_models")
|
||||
| _provider_set("darkbloom_models")
|
||||
| _provider_set("openinfer_models")
|
||||
| _provider_set("v0_models")
|
||||
| _provider_set("morph_models")
|
||||
| _provider_set("opencode_go_models")
|
||||
| _provider_set("lambda_ai_models")
|
||||
| _provider_set("inception_models")
|
||||
| _provider_set("black_forest_labs_models")
|
||||
| _provider_set("recraft_models")
|
||||
| _provider_set("cometapi_models")
|
||||
| _provider_set("oci_models")
|
||||
| _provider_set("heroku_models")
|
||||
| _provider_set("vercel_ai_gateway_models")
|
||||
| _provider_set("edenai_models")
|
||||
| _provider_set("volcengine_models")
|
||||
| _provider_set("wandb_models")
|
||||
| _provider_set("ovhcloud_models")
|
||||
| _provider_set("lemonade_models")
|
||||
| _provider_set("docker_model_runner_models")
|
||||
| _provider_set("reducto_models")
|
||||
| _provider_set("bedrock_mantle_models")
|
||||
| _provider_set("coralbricks_models")
|
||||
| set(clarifai_models)
|
||||
)
|
||||
|
||||
|
|
@ -1172,128 +742,13 @@ model_list_set = set(model_list)
|
|||
# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
|
||||
|
||||
|
||||
def _build_models_by_provider() -> dict:
|
||||
return {
|
||||
"openai": open_ai_chat_completion_models | open_ai_text_completion_models,
|
||||
"text-completion-openai": open_ai_text_completion_models,
|
||||
"cohere": cohere_models | cohere_chat_models,
|
||||
"cohere_chat": cohere_chat_models,
|
||||
"anthropic": anthropic_models,
|
||||
"replicate": replicate_models,
|
||||
"huggingface": huggingface_models,
|
||||
"together_ai": together_ai_models,
|
||||
"baseten": baseten_models,
|
||||
"openrouter": openrouter_models,
|
||||
"vercel_ai_gateway": vercel_ai_gateway_models,
|
||||
"edenai": edenai_models,
|
||||
"datarobot": datarobot_models,
|
||||
"vertex_ai": vertex_chat_models
|
||||
| vertex_code_chat_models
|
||||
| vertex_text_models
|
||||
| vertex_code_text_models
|
||||
| vertex_anthropic_models
|
||||
| vertex_vision_models
|
||||
| vertex_language_models
|
||||
| vertex_deepseek_models
|
||||
| vertex_minimax_models
|
||||
| vertex_moonshot_models
|
||||
| vertex_zai_models,
|
||||
"ai21": ai21_models | ai21_chat_models,
|
||||
"bedrock": bedrock_models | bedrock_converse_models,
|
||||
"petals": petals_models,
|
||||
"ollama": ollama_models,
|
||||
"ollama_chat": ollama_models,
|
||||
"deepinfra": deepinfra_models,
|
||||
"perplexity": perplexity_models,
|
||||
"maritalk": maritalk_models,
|
||||
"watsonx": watsonx_models,
|
||||
"gemini": gemini_models,
|
||||
"fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models,
|
||||
"aleph_alpha": aleph_alpha_models,
|
||||
"text-completion-codestral": text_completion_codestral_models,
|
||||
"text-completion-inception": text_completion_inception_models,
|
||||
"xai": xai_models,
|
||||
"zai": zai_models,
|
||||
"fal_ai": fal_ai_models,
|
||||
"deepseek": deepseek_models,
|
||||
"tencent": tencent_models,
|
||||
"runwayml": runwayml_models,
|
||||
"mistral": mistral_chat_models,
|
||||
"azure_ai": azure_ai_models,
|
||||
"voyage": voyage_models,
|
||||
"infinity": infinity_models,
|
||||
"databricks": databricks_models,
|
||||
"cloudflare": cloudflare_models,
|
||||
"codestral": codestral_models,
|
||||
"nlp_cloud": nlp_cloud_models,
|
||||
"friendliai": friendliai_models,
|
||||
"palm": palm_models,
|
||||
"groq": groq_models,
|
||||
"azure": azure_models | azure_text_models,
|
||||
"azure_anthropic": azure_anthropic_models,
|
||||
"azure_text": azure_text_models,
|
||||
"anyscale": anyscale_models,
|
||||
"cerebras": cerebras_models,
|
||||
"nadir": nadir_models,
|
||||
"galadriel": galadriel_models,
|
||||
"nvidia_nim": nvidia_nim_models,
|
||||
"nvidia_riva": nvidia_riva_models,
|
||||
"soniox": soniox_models,
|
||||
"sambanova": sambanova_models | sambanova_embedding_models,
|
||||
"scaledown": scaledown_models,
|
||||
"novita": novita_models,
|
||||
"nebius": nebius_models | nebius_embedding_models,
|
||||
"aiml": aiml_models,
|
||||
"assemblyai": assemblyai_models,
|
||||
"jina_ai": jina_ai_models,
|
||||
"snowflake": snowflake_models,
|
||||
"gradient_ai": gradient_ai_models,
|
||||
"meta_llama": llama_models,
|
||||
"nscale": nscale_models,
|
||||
"featherless_ai": featherless_ai_models,
|
||||
"deepgram": deepgram_models,
|
||||
"elevenlabs": elevenlabs_models,
|
||||
"heroku": heroku_models,
|
||||
"dashscope": dashscope_models,
|
||||
"qwencloud": qwencloud_models,
|
||||
"qwen_ai_platform": qwen_ai_platform_models,
|
||||
"modelscope": modelscope_models,
|
||||
"moonshot": moonshot_models,
|
||||
"publicai": publicai_models,
|
||||
"darkbloom": darkbloom_models,
|
||||
"openinfer": openinfer_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
"inception": inception_models,
|
||||
"hyperbolic": hyperbolic_models,
|
||||
"black_forest_labs": black_forest_labs_models,
|
||||
"recraft": recraft_models,
|
||||
"cometapi": cometapi_models,
|
||||
"oci": oci_models,
|
||||
"volcengine": volcengine_models,
|
||||
"wandb": wandb_models,
|
||||
"ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
|
||||
"lemonade": lemonade_models,
|
||||
"clarifai": clarifai_models,
|
||||
"amazon_nova": amazon_nova_models,
|
||||
"stability": stability_models,
|
||||
"github_copilot": github_copilot_models,
|
||||
"microsoft_365_copilot": microsoft_365_copilot_models,
|
||||
"chatgpt": chatgpt_models,
|
||||
"minimax": minimax_models,
|
||||
"aws_polly": aws_polly_models,
|
||||
"transcribe": transcribe_models,
|
||||
"gigachat": gigachat_models,
|
||||
"llamagate": llamagate_models,
|
||||
"coralbricks": coralbricks_models,
|
||||
"reducto": reducto_models,
|
||||
"bedrock_mantle": bedrock_mantle_models,
|
||||
}
|
||||
|
||||
|
||||
models_by_provider: dict = _build_models_by_provider()
|
||||
models_by_provider: Final = ModelsByProvider()
|
||||
|
||||
|
||||
def add_known_models(model_cost_map: Optional[Dict] = None) -> None:
|
||||
"""Kept for callers. The provider sets are derived from the live map on read (decision 18)"""
|
||||
|
||||
# mapping for those models which have larger equivalents
|
||||
longer_context_model_fallback_dict: dict = {
|
||||
|
|
@ -1323,11 +778,11 @@ all_embedding_models = (
|
|||
open_ai_embedding_models
|
||||
| set(cohere_embedding_models)
|
||||
| set(bedrock_embedding_models)
|
||||
| vertex_embedding_models
|
||||
| fireworks_ai_embedding_models
|
||||
| nebius_embedding_models
|
||||
| sambanova_embedding_models
|
||||
| ovhcloud_embedding_models
|
||||
| _provider_set("vertex_embedding_models")
|
||||
| _provider_set("fireworks_ai_embedding_models")
|
||||
| _provider_set("nebius_embedding_models")
|
||||
| _provider_set("sambanova_embedding_models")
|
||||
| _provider_set("ovhcloud_embedding_models")
|
||||
)
|
||||
|
||||
####### IMAGE GENERATION MODELS ###################
|
||||
|
|
@ -2429,6 +1884,12 @@ if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes",
|
|||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name in _LEGACY_PROVIDER_SETS:
|
||||
return _provider_set(name)
|
||||
return _lazy_getattr(name)
|
||||
|
||||
|
||||
def _lazy_getattr(name: str) -> Any:
|
||||
"""Lazy import handler with cached registry for improved performance."""
|
||||
global _async_client_cleanup_registered
|
||||
# Register async client cleanup on first access (only once)
|
||||
|
|
|
|||
413
litellm/_catalog/provider_sets.py
Normal file
413
litellm/_catalog/provider_sets.py
Normal file
|
|
@ -0,0 +1,413 @@
|
|||
"""The provider model sets, derived from the live map on read (decision 18). The only place that knows provider names.
|
||||
``LEGACY_NAMES`` names each of main's module-level sets and the raw ``litellm_provider`` strings it holds, ``FILTERS``
|
||||
and ``BARE_KEYS`` carry the rules main's branch chain applied, and ``ALIASES`` is main's ``models_by_provider``"""
|
||||
|
||||
import datetime
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from litellm._catalog import CATALOG
|
||||
|
||||
LEGACY_NAMES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
|
||||
{
|
||||
"open_ai_chat_completion_models": ("openai",),
|
||||
"open_ai_text_completion_models": ("text-completion-openai",),
|
||||
"cohere_models": ("cohere",),
|
||||
"cohere_chat_models": ("cohere_chat",),
|
||||
"mistral_chat_models": ("mistral",),
|
||||
"text_completion_codestral_models": ("text-completion-codestral",),
|
||||
"text_completion_inception_models": ("text-completion-inception",),
|
||||
"anthropic_models": ("anthropic",),
|
||||
"openrouter_models": ("openrouter",),
|
||||
"datarobot_models": ("datarobot",),
|
||||
"vertex_language_models": ("vertex_ai-language-models",),
|
||||
"vertex_vision_models": ("vertex_ai-vision-models",),
|
||||
"vertex_chat_models": ("vertex_ai-chat-models",),
|
||||
"vertex_code_chat_models": ("vertex_ai-code-chat-models",),
|
||||
"vertex_ai_image_models": ("vertex_ai-image-models",),
|
||||
"vertex_ai_video_models": ("vertex_ai-video-models",),
|
||||
"vertex_text_models": ("vertex_ai-text-models",),
|
||||
"vertex_code_text_models": ("vertex_ai-code-text-models",),
|
||||
"vertex_embedding_models": ("vertex_ai-embedding-models",),
|
||||
"vertex_anthropic_models": ("vertex_ai-anthropic_models",),
|
||||
"vertex_llama3_models": ("vertex_ai-llama_models",),
|
||||
"vertex_deepseek_models": ("vertex_ai-deepseek_models",),
|
||||
"vertex_ai_ai21_models": ("vertex_ai-ai21_models",),
|
||||
"vertex_mistral_models": ("vertex_ai-mistral_models",),
|
||||
"vertex_openai_models": ("vertex_ai-openai_models",),
|
||||
"vertex_minimax_models": ("vertex_ai-minimax_models",),
|
||||
"vertex_moonshot_models": ("vertex_ai-moonshot_models",),
|
||||
"vertex_zai_models": ("vertex_ai-zai_models",),
|
||||
"ai21_models": ("ai21",),
|
||||
"ai21_chat_models": ("ai21",),
|
||||
"nlp_cloud_models": ("nlp_cloud",),
|
||||
"aleph_alpha_models": ("aleph_alpha",),
|
||||
"bedrock_models": ("bedrock",),
|
||||
"bedrock_converse_models": ("bedrock_converse",),
|
||||
"fal_ai_models": ("fal_ai",),
|
||||
"fireworks_ai_models": ("fireworks_ai",),
|
||||
"baseten_models": ("baseten",),
|
||||
"fireworks_ai_embedding_models": ("fireworks_ai-embedding-models",),
|
||||
"deepinfra_models": ("deepinfra",),
|
||||
"perplexity_models": ("perplexity",),
|
||||
"watsonx_models": ("watsonx",),
|
||||
"gemini_models": ("gemini",),
|
||||
"xai_models": ("xai",),
|
||||
"zai_models": ("zai",),
|
||||
"deepseek_models": ("deepseek",),
|
||||
"tencent_models": ("tencent",),
|
||||
"runwayml_models": ("runwayml",),
|
||||
"azure_ai_models": ("azure_ai",),
|
||||
"jina_ai_models": ("jina_ai",),
|
||||
"voyage_models": ("voyage",),
|
||||
"infinity_models": ("infinity",),
|
||||
"heroku_models": ("heroku",),
|
||||
"databricks_models": ("databricks",),
|
||||
"cloudflare_models": ("cloudflare",),
|
||||
"codestral_models": ("codestral",),
|
||||
"friendliai_models": ("friendliai",),
|
||||
"featherless_ai_models": ("featherless_ai",),
|
||||
"palm_models": ("palm",),
|
||||
"groq_models": ("groq",),
|
||||
"azure_models": ("azure",),
|
||||
"azure_anthropic_models": ("azure_anthropic",),
|
||||
"azure_text_models": ("azure_text",),
|
||||
"anyscale_models": ("anyscale",),
|
||||
"cerebras_models": ("cerebras",),
|
||||
"nadir_models": ("nadir",),
|
||||
"galadriel_models": ("galadriel",),
|
||||
"nvidia_nim_models": ("nvidia_nim",),
|
||||
"nvidia_riva_models": ("nvidia_riva",),
|
||||
"soniox_models": ("soniox",),
|
||||
"sambanova_models": ("sambanova",),
|
||||
"scaledown_models": ("scaledown",),
|
||||
"sambanova_embedding_models": ("sambanova-embedding-models",),
|
||||
"novita_models": ("novita",),
|
||||
"assemblyai_models": ("assemblyai",),
|
||||
"snowflake_models": ("snowflake",),
|
||||
"gradient_ai_models": ("gradient_ai",),
|
||||
"llama_models": ("meta_llama",),
|
||||
"nscale_models": ("nscale",),
|
||||
"nebius_models": ("nebius-chat-models",),
|
||||
"nebius_embedding_models": ("nebius-embedding-models",),
|
||||
"aiml_models": ("aiml",),
|
||||
"deepgram_models": ("deepgram",),
|
||||
"elevenlabs_models": ("elevenlabs",),
|
||||
"dashscope_models": ("dashscope",),
|
||||
"qwencloud_models": ("qwencloud",),
|
||||
"qwen_ai_platform_models": ("qwen_ai_platform",),
|
||||
"moonshot_models": ("moonshot",),
|
||||
"publicai_models": ("publicai",),
|
||||
"darkbloom_models": ("darkbloom",),
|
||||
"openinfer_models": ("openinfer",),
|
||||
"v0_models": ("v0",),
|
||||
"morph_models": ("morph",),
|
||||
"opencode_go_models": ("opencode_go",),
|
||||
"lambda_ai_models": ("lambda_ai",),
|
||||
"inception_models": ("inception",),
|
||||
"hyperbolic_models": ("hyperbolic",),
|
||||
"black_forest_labs_models": ("black_forest_labs",),
|
||||
"recraft_models": ("recraft",),
|
||||
"cometapi_models": ("cometapi",),
|
||||
"oci_models": ("oci",),
|
||||
"vercel_ai_gateway_models": ("vercel_ai_gateway",),
|
||||
"edenai_models": ("edenai",),
|
||||
"volcengine_models": ("volcengine",),
|
||||
"wandb_models": ("wandb",),
|
||||
"ovhcloud_models": ("ovhcloud",),
|
||||
"ovhcloud_embedding_models": ("ovhcloud-embedding-models",),
|
||||
"lemonade_models": ("lemonade",),
|
||||
"docker_model_runner_models": ("docker_model_runner",),
|
||||
"amazon_nova_models": ("amazon_nova",),
|
||||
"stability_models": ("stability",),
|
||||
"github_copilot_models": ("github_copilot",),
|
||||
"microsoft_365_copilot_models": ("microsoft_365_copilot",),
|
||||
"chatgpt_models": ("chatgpt",),
|
||||
"minimax_models": ("minimax",),
|
||||
"aws_polly_models": ("aws_polly",),
|
||||
"transcribe_models": ("transcribe",),
|
||||
"gigachat_models": ("gigachat",),
|
||||
"llamagate_models": ("llamagate",),
|
||||
"coralbricks_models": ("coralbricks",),
|
||||
"reducto_models": ("reducto",),
|
||||
"bedrock_mantle_models": ("bedrock_mantle",),
|
||||
}
|
||||
)
|
||||
|
||||
_BEDROCK_PRICING_ONLY: Final = re.compile(r"^bedrock/[a-zA-Z0-9_-]+/.+$")
|
||||
|
||||
|
||||
def is_bedrock_pricing_only_model(key: str) -> bool:
|
||||
"""Keys of the pattern ``bedrock/<region>/<model>`` and commitment keys, priced for lookup only"""
|
||||
return "month-commitment" in key or _BEDROCK_PRICING_ONLY.match(key) is not None
|
||||
|
||||
|
||||
def is_openai_finetune_model(key: str) -> bool:
|
||||
"""``ft:<model>`` keys, priced for lookup only"""
|
||||
return key.startswith("ft:") and not key.count(":") > 1
|
||||
|
||||
|
||||
def _is_past_deprecation_date(row: Mapping[str, object]) -> bool:
|
||||
deprecation_date: Final = row.get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return datetime.date.fromisoformat(deprecation_date) <= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
FILTERS: Final[Mapping[str, Callable[[str, Mapping[str, object]], bool]]] = MappingProxyType(
|
||||
{
|
||||
"open_ai_chat_completion_models": lambda key, row: not is_openai_finetune_model(key),
|
||||
"ai21_chat_models": lambda key, row: row.get("mode") == "chat",
|
||||
"ai21_models": lambda key, row: row.get("mode") != "chat",
|
||||
"bedrock_models": lambda key, row: row.get("mode") != "guardrail" and not is_bedrock_pricing_only_model(key),
|
||||
"fireworks_ai_models": lambda key, row: "-to-" not in key and "fireworks-ai-default" not in key,
|
||||
"fireworks_ai_embedding_models": lambda key, row: "-to-" not in key,
|
||||
"baseten_models": lambda key, row: not _is_past_deprecation_date(row),
|
||||
}
|
||||
)
|
||||
|
||||
BARE_KEYS: Final = frozenset(
|
||||
{
|
||||
"vertex_ai_ai21_models",
|
||||
"vertex_ai_image_models",
|
||||
"vertex_ai_video_models",
|
||||
"vertex_anthropic_models",
|
||||
"vertex_deepseek_models",
|
||||
"vertex_llama3_models",
|
||||
"vertex_minimax_models",
|
||||
"vertex_mistral_models",
|
||||
"vertex_moonshot_models",
|
||||
"vertex_openai_models",
|
||||
"vertex_zai_models",
|
||||
}
|
||||
)
|
||||
"""Sets main filled with the ``vertex_ai/`` prefix stripped from each key"""
|
||||
|
||||
STATIC: Final = frozenset(
|
||||
{
|
||||
"clarifai_models",
|
||||
"huggingface_models",
|
||||
"maritalk_models",
|
||||
"ollama_models",
|
||||
"petals_models",
|
||||
"replicate_models",
|
||||
"together_ai_models",
|
||||
}
|
||||
)
|
||||
"""Hand lists of models the map has no rows for, read from ``litellm`` as they are"""
|
||||
|
||||
ALIASES: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
|
||||
{
|
||||
"openai": ("open_ai_chat_completion_models", "open_ai_text_completion_models"),
|
||||
"text-completion-openai": ("open_ai_text_completion_models",),
|
||||
"cohere": ("cohere_models", "cohere_chat_models"),
|
||||
"cohere_chat": ("cohere_chat_models",),
|
||||
"anthropic": ("anthropic_models",),
|
||||
"replicate": ("replicate_models",),
|
||||
"huggingface": ("huggingface_models",),
|
||||
"together_ai": ("together_ai_models",),
|
||||
"baseten": ("baseten_models",),
|
||||
"openrouter": ("openrouter_models",),
|
||||
"vercel_ai_gateway": ("vercel_ai_gateway_models",),
|
||||
"edenai": ("edenai_models",),
|
||||
"datarobot": ("datarobot_models",),
|
||||
"vertex_ai": (
|
||||
"vertex_chat_models",
|
||||
"vertex_code_chat_models",
|
||||
"vertex_text_models",
|
||||
"vertex_code_text_models",
|
||||
"vertex_anthropic_models",
|
||||
"vertex_vision_models",
|
||||
"vertex_language_models",
|
||||
"vertex_deepseek_models",
|
||||
"vertex_minimax_models",
|
||||
"vertex_moonshot_models",
|
||||
"vertex_zai_models",
|
||||
),
|
||||
"ai21": ("ai21_models", "ai21_chat_models"),
|
||||
"bedrock": ("bedrock_models", "bedrock_converse_models"),
|
||||
"petals": ("petals_models",),
|
||||
"ollama": ("ollama_models",),
|
||||
"ollama_chat": ("ollama_models",),
|
||||
"deepinfra": ("deepinfra_models",),
|
||||
"perplexity": ("perplexity_models",),
|
||||
"maritalk": ("maritalk_models",),
|
||||
"watsonx": ("watsonx_models",),
|
||||
"gemini": ("gemini_models",),
|
||||
"fireworks_ai": ("fireworks_ai_models", "fireworks_ai_embedding_models"),
|
||||
"aleph_alpha": ("aleph_alpha_models",),
|
||||
"text-completion-codestral": ("text_completion_codestral_models",),
|
||||
"text-completion-inception": ("text_completion_inception_models",),
|
||||
"xai": ("xai_models",),
|
||||
"zai": ("zai_models",),
|
||||
"fal_ai": ("fal_ai_models",),
|
||||
"deepseek": ("deepseek_models",),
|
||||
"tencent": ("tencent_models",),
|
||||
"runwayml": ("runwayml_models",),
|
||||
"mistral": ("mistral_chat_models",),
|
||||
"azure_ai": ("azure_ai_models",),
|
||||
"voyage": ("voyage_models",),
|
||||
"infinity": ("infinity_models",),
|
||||
"databricks": ("databricks_models",),
|
||||
"cloudflare": ("cloudflare_models",),
|
||||
"codestral": ("codestral_models",),
|
||||
"nlp_cloud": ("nlp_cloud_models",),
|
||||
"friendliai": ("friendliai_models",),
|
||||
"palm": ("palm_models",),
|
||||
"groq": ("groq_models",),
|
||||
"azure": ("azure_models", "azure_text_models"),
|
||||
"azure_anthropic": ("azure_anthropic_models",),
|
||||
"azure_text": ("azure_text_models",),
|
||||
"anyscale": ("anyscale_models",),
|
||||
"cerebras": ("cerebras_models",),
|
||||
"nadir": ("nadir_models",),
|
||||
"galadriel": ("galadriel_models",),
|
||||
"nvidia_nim": ("nvidia_nim_models",),
|
||||
"nvidia_riva": ("nvidia_riva_models",),
|
||||
"soniox": ("soniox_models",),
|
||||
"sambanova": ("sambanova_models", "sambanova_embedding_models"),
|
||||
"scaledown": ("scaledown_models",),
|
||||
"novita": ("novita_models",),
|
||||
"nebius": ("nebius_models", "nebius_embedding_models"),
|
||||
"aiml": ("aiml_models",),
|
||||
"assemblyai": ("assemblyai_models",),
|
||||
"jina_ai": ("jina_ai_models",),
|
||||
"snowflake": ("snowflake_models",),
|
||||
"gradient_ai": ("gradient_ai_models",),
|
||||
"meta_llama": ("llama_models",),
|
||||
"nscale": ("nscale_models",),
|
||||
"featherless_ai": ("featherless_ai_models",),
|
||||
"deepgram": ("deepgram_models",),
|
||||
"elevenlabs": ("elevenlabs_models",),
|
||||
"heroku": ("heroku_models",),
|
||||
"dashscope": ("dashscope_models",),
|
||||
"qwencloud": ("qwencloud_models",),
|
||||
"qwen_ai_platform": ("qwen_ai_platform_models",),
|
||||
"modelscope": ("modelscope_models",),
|
||||
"moonshot": ("moonshot_models",),
|
||||
"publicai": ("publicai_models",),
|
||||
"darkbloom": ("darkbloom_models",),
|
||||
"openinfer": ("openinfer_models",),
|
||||
"v0": ("v0_models",),
|
||||
"morph": ("morph_models",),
|
||||
"opencode_go": ("opencode_go_models",),
|
||||
"lambda_ai": ("lambda_ai_models",),
|
||||
"inception": ("inception_models",),
|
||||
"hyperbolic": ("hyperbolic_models",),
|
||||
"black_forest_labs": ("black_forest_labs_models",),
|
||||
"recraft": ("recraft_models",),
|
||||
"cometapi": ("cometapi_models",),
|
||||
"oci": ("oci_models",),
|
||||
"volcengine": ("volcengine_models",),
|
||||
"wandb": ("wandb_models",),
|
||||
"ovhcloud": ("ovhcloud_models", "ovhcloud_embedding_models"),
|
||||
"lemonade": ("lemonade_models",),
|
||||
"clarifai": ("clarifai_models",),
|
||||
"amazon_nova": ("amazon_nova_models",),
|
||||
"stability": ("stability_models",),
|
||||
"github_copilot": ("github_copilot_models",),
|
||||
"microsoft_365_copilot": ("microsoft_365_copilot_models",),
|
||||
"chatgpt": ("chatgpt_models",),
|
||||
"minimax": ("minimax_models",),
|
||||
"aws_polly": ("aws_polly_models",),
|
||||
"transcribe": ("transcribe_models",),
|
||||
"gigachat": ("gigachat_models",),
|
||||
"llamagate": ("llamagate_models",),
|
||||
"coralbricks": ("coralbricks_models",),
|
||||
"reducto": ("reducto_models",),
|
||||
"bedrock_mantle": ("bedrock_mantle_models",),
|
||||
}
|
||||
)
|
||||
"""main's ``models_by_provider``, each provider the union of the named sets"""
|
||||
|
||||
_SEEDS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{"bedrock_converse_models": "BEDROCK_CONVERSE_MODELS", "wandb_models": "WANDB_MODELS"}
|
||||
)
|
||||
|
||||
|
||||
OPAQUE_KEYS: Final[set[str]] = set() # mutable-ok: register_model adds router deployment ids, never removed
|
||||
"""Keys registered as opaque ids (``add_to_provider_model_sets=False``), which no provider set lists"""
|
||||
|
||||
|
||||
def _litellm() -> object:
|
||||
return sys.modules["litellm"]
|
||||
|
||||
|
||||
def _rows() -> Mapping[str, Mapping[str, object]]:
|
||||
import litellm
|
||||
|
||||
return litellm.model_cost
|
||||
|
||||
|
||||
def _derived(name: str) -> frozenset[str]:
|
||||
rows: Final = _rows()
|
||||
keep: Final = FILTERS.get(name, lambda key, row: True)
|
||||
bare: Final = name in BARE_KEYS
|
||||
keys: Final = (
|
||||
key
|
||||
for provider in LEGACY_NAMES[name]
|
||||
for key in CATALOG.models_for(provider) # comprehension-ok: a set's providers, then each provider's keys
|
||||
if keep(key, rows.get(key) or {}) and key not in OPAQUE_KEYS
|
||||
)
|
||||
seed: Final = _SEEDS.get(name)
|
||||
seeded: Final = frozenset(getattr(_litellm(), seed)) if seed is not None else frozenset()
|
||||
return frozenset(key.replace("vertex_ai/", "") if bare else key for key in keys) | seeded
|
||||
|
||||
|
||||
def legacy_set(name: str) -> frozenset[str]:
|
||||
"""One of main's 121 module-level provider sets, as the live map has it now. Read only"""
|
||||
return _derived(name)
|
||||
|
||||
|
||||
def contains(name: str, model: str) -> bool:
|
||||
"""``model in litellm.<name>`` without building the set, for the request path. The same rows, filters, bare keys
|
||||
and seeds as ``legacy_set``. A set assigned to ``litellm.<name>`` answers instead"""
|
||||
assigned: Final = vars(_litellm()).get(name)
|
||||
if assigned is not None:
|
||||
return model in assigned
|
||||
rows: Final = _rows()
|
||||
keep: Final = FILTERS.get(name, lambda key, row: True)
|
||||
bare: Final = name in BARE_KEYS
|
||||
if bare and "vertex_ai/" in model:
|
||||
return False
|
||||
keys: Final = (model, f"vertex_ai/{model}") if bare else (model,)
|
||||
in_map: Final = any(
|
||||
isinstance(row := rows.get(key), Mapping)
|
||||
and row.get("litellm_provider") in LEGACY_NAMES[name]
|
||||
and key not in OPAQUE_KEYS
|
||||
and keep(key, row)
|
||||
for key in keys
|
||||
)
|
||||
seed: Final = _SEEDS.get(name)
|
||||
return in_map or (seed is not None and model in getattr(_litellm(), seed))
|
||||
|
||||
|
||||
def models_for(provider: str) -> frozenset[str]:
|
||||
"""The models of ``provider`` as ``models_by_provider`` lists them"""
|
||||
names: Final = ALIASES.get(provider)
|
||||
if names is None:
|
||||
return CATALOG.models_for(provider) - OPAQUE_KEYS
|
||||
return frozenset().union(
|
||||
*(legacy_set(name) if name in LEGACY_NAMES else frozenset(getattr(_litellm(), name)) for name in names)
|
||||
)
|
||||
|
||||
|
||||
class ModelsByProvider(Mapping[str, frozenset[str]]):
|
||||
"""``litellm.models_by_provider``, read only. Main's providers, then every raw provider string in the map"""
|
||||
|
||||
def __getitem__(self, provider: str) -> frozenset[str]:
|
||||
if provider in ALIASES or provider in CATALOG.providers():
|
||||
return models_for(provider)
|
||||
raise KeyError(provider)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter((*ALIASES, *(provider for provider in CATALOG.providers() if provider not in ALIASES)))
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(ALIASES) + sum(1 for provider in CATALOG.providers() if provider not in ALIASES)
|
||||
|
|
@ -160,11 +160,9 @@ def _install(
|
|||
|
||||
|
||||
def _refresh_derived_state() -> None:
|
||||
import litellm
|
||||
from litellm import utils
|
||||
|
||||
utils._invalidate_model_cost_lowercase_map() # pyright: ignore[reportPrivateUsage] # required cache invalidation
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
|
||||
|
||||
def reload_model_cost_map(source: Source, catalog: Catalog = CATALOG) -> Installed | Unavailable:
|
||||
|
|
|
|||
|
|
@ -248,6 +248,8 @@ class Catalog:
|
|||
def resolve(self, model: str, provider: str | None = None) -> Resolved | Missing | Invalid: ...
|
||||
def lookup_key(self, name: str) -> str | None: ...
|
||||
def provider_keys(self, provider: str) -> list[str]: ...
|
||||
def models_for(self, provider: str) -> frozenset[str]: ...
|
||||
def providers(self) -> list[str]: ...
|
||||
def mark_index_stale(self) -> None: ...
|
||||
def register(
|
||||
self, rows: Mapping[str, Mapping[str, object]], *, persist: bool, provider: str | None = None
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ import litellm.litellm_core_utils
|
|||
|
||||
# audio_utils.utils is lazy-loaded - only imported when needed for transcription calls
|
||||
import litellm.litellm_core_utils.json_validation_rule
|
||||
from litellm._catalog.provider_sets import contains as _in_provider_set
|
||||
from litellm._internal_context import is_internal_call
|
||||
from litellm._lazy_imports import (
|
||||
_get_modified_max_tokens,
|
||||
|
|
@ -2635,7 +2636,7 @@ def _return_openai_tokenizer(model: str) -> SelectTokenizerResponse:
|
|||
|
||||
|
||||
def uses_anthropic_tokenizer(model: str) -> bool:
|
||||
return model in litellm.anthropic_models and "claude-3" not in model
|
||||
return _in_provider_set("anthropic_models", model) and "claude-3" not in model
|
||||
|
||||
|
||||
HuggingFaceTokenizerKind = Literal["cohere", "anthropic", "deepseek", "llama2", "llama3"]
|
||||
|
|
@ -2651,7 +2652,7 @@ _BUNDLED_TOKENIZER_LOAD_LOCKS: Final[Mapping[HuggingFaceTokenizerKind, threading
|
|||
|
||||
def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None:
|
||||
"""Which HuggingFace tokenizer `token_counter` selects for a model; `None` means tiktoken."""
|
||||
if model in litellm.cohere_models and "command-r" in model:
|
||||
if _in_provider_set("cohere_models", model) and "command-r" in model:
|
||||
return "cohere"
|
||||
if uses_anthropic_tokenizer(model):
|
||||
return "anthropic"
|
||||
|
|
@ -3656,9 +3657,10 @@ def register_model(
|
|||
that provider, so a deployment id that happens to equal another provider's
|
||||
catalog key stays its own provider-less entry instead of merging into it.
|
||||
|
||||
``add_to_provider_model_sets`` adds the keys to the per-provider model sets
|
||||
behind provider resolution and wildcard model listing. Pass False when the
|
||||
keys are opaque ids rather than model names (e.g. router deployment ids).
|
||||
The provider model sets are derived from ``litellm.model_cost`` on read
|
||||
(decision 18). Pass ``add_to_provider_model_sets=False`` when the keys are
|
||||
opaque ids rather than model names (e.g. router deployment ids), so no set
|
||||
lists them.
|
||||
"""
|
||||
|
||||
loaded_model_cost = {}
|
||||
|
|
@ -3685,12 +3687,12 @@ def register_model(
|
|||
"register_model: model=%s has custom pricing but not in built-in cost map and no prefix/region variant matched; cache_creation_input_token_cost and cache_read_input_token_cost will default to 0 for this model (input/output cost tracking is unaffected). To track cache cost, add them to model_info",
|
||||
warning_display_name or key,
|
||||
)
|
||||
for key, value in loaded_model_cost.items():
|
||||
for key in loaded_model_cost:
|
||||
verbose_logger.debug("added/updated model=%s in litellm.model_cost", key)
|
||||
if add_to_provider_model_sets and value.get("litellm_provider") == "openrouter":
|
||||
litellm.openrouter_models.add(str(key).split("/", 1)[-1])
|
||||
if add_to_provider_model_sets:
|
||||
litellm.add_known_models(loaded_model_cost)
|
||||
if not add_to_provider_model_sets:
|
||||
from litellm._catalog.provider_sets import OPAQUE_KEYS
|
||||
|
||||
OPAQUE_KEYS.update(str(key) for key in loaded_model_cost)
|
||||
return model_cost
|
||||
|
||||
|
||||
|
|
@ -4864,12 +4866,12 @@ def get_optional_params(
|
|||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai" and (
|
||||
model in litellm.vertex_chat_models
|
||||
or model in litellm.vertex_code_chat_models
|
||||
or model in litellm.vertex_text_models
|
||||
or model in litellm.vertex_code_text_models
|
||||
or model in litellm.vertex_language_models
|
||||
or model in litellm.vertex_vision_models
|
||||
_in_provider_set("vertex_chat_models", model)
|
||||
or _in_provider_set("vertex_code_chat_models", model)
|
||||
or _in_provider_set("vertex_text_models", model)
|
||||
or _in_provider_set("vertex_code_text_models", model)
|
||||
or _in_provider_set("vertex_language_models", model)
|
||||
or _in_provider_set("vertex_vision_models", model)
|
||||
):
|
||||
optional_params = litellm.VertexGeminiConfig().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
|
|
@ -4900,7 +4902,7 @@ def get_optional_params(
|
|||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
if model in litellm.vertex_mistral_models:
|
||||
if _in_provider_set("vertex_mistral_models", model):
|
||||
if "codestral" in model:
|
||||
optional_params = litellm.CodestralTextCompletionConfig().map_openai_params(
|
||||
model=model,
|
||||
|
|
@ -4915,7 +4917,7 @@ def get_optional_params(
|
|||
optional_params=optional_params,
|
||||
drop_params=bool(drop_params),
|
||||
)
|
||||
elif model in litellm.vertex_ai_ai21_models:
|
||||
elif _in_provider_set("vertex_ai_ai21_models", model):
|
||||
optional_params = litellm.VertexAIAi21Config().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
|
|
@ -6109,10 +6111,10 @@ def get_model_info_helper(
|
|||
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai_beta":
|
||||
custom_llm_provider = "vertex_ai"
|
||||
if custom_llm_provider is not None and custom_llm_provider == "vertex_ai":
|
||||
if "meta/" + model in litellm.vertex_llama3_models:
|
||||
if _in_provider_set("vertex_llama3_models", "meta/" + model):
|
||||
model = "meta/" + model
|
||||
elif (
|
||||
model + "@latest" in litellm.vertex_mistral_models or model + "@latest" in litellm.vertex_ai_ai21_models
|
||||
elif _in_provider_set("vertex_mistral_models", model + "@latest") or _in_provider_set(
|
||||
"vertex_ai_ai21_models", model + "@latest"
|
||||
):
|
||||
model = model + "@latest"
|
||||
##########################
|
||||
|
|
@ -6845,8 +6847,8 @@ def validate_environment(
|
|||
else:
|
||||
## openai - chatcompletion + text completion
|
||||
if (
|
||||
model in litellm.open_ai_chat_completion_models
|
||||
or model in litellm.open_ai_text_completion_models
|
||||
_in_provider_set("open_ai_chat_completion_models", model)
|
||||
or _in_provider_set("open_ai_text_completion_models", model)
|
||||
or model in litellm.open_ai_embedding_models
|
||||
or model in litellm.openai_image_generation_models
|
||||
or model.startswith("gpt-image")
|
||||
|
|
@ -6856,13 +6858,13 @@ def validate_environment(
|
|||
else:
|
||||
missing_keys.append("OPENAI_API_KEY")
|
||||
## anthropic
|
||||
elif model in litellm.anthropic_models:
|
||||
elif _in_provider_set("anthropic_models", model):
|
||||
if "ANTHROPIC_API_KEY" in os.environ or "ANTHROPIC_AUTH_TOKEN" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("ANTHROPIC_API_KEY")
|
||||
## cohere
|
||||
elif model in litellm.cohere_models:
|
||||
elif _in_provider_set("cohere_models", model):
|
||||
if "COHERE_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
|
|
@ -6874,33 +6876,33 @@ def validate_environment(
|
|||
else:
|
||||
missing_keys.append("REPLICATE_API_KEY")
|
||||
## openrouter
|
||||
elif model in litellm.openrouter_models:
|
||||
elif _in_provider_set("openrouter_models", model):
|
||||
if "OPENROUTER_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("OPENROUTER_API_KEY")
|
||||
## vercel_ai_gateway
|
||||
elif model in litellm.vercel_ai_gateway_models:
|
||||
elif _in_provider_set("vercel_ai_gateway_models", model):
|
||||
if "VERCEL_AI_GATEWAY_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("VERCEL_AI_GATEWAY_API_KEY")
|
||||
## edenai
|
||||
elif model in litellm.edenai_models:
|
||||
elif _in_provider_set("edenai_models", model):
|
||||
if "EDENAI_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("EDENAI_API_KEY")
|
||||
## datarobot
|
||||
elif model in litellm.datarobot_models:
|
||||
elif _in_provider_set("datarobot_models", model):
|
||||
if "DATAROBOT_API_TOKEN" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("DATAROBOT_API_TOKEN")
|
||||
## vertex - text + chat models
|
||||
elif (
|
||||
model in litellm.vertex_chat_models
|
||||
or model in litellm.vertex_text_models
|
||||
_in_provider_set("vertex_chat_models", model)
|
||||
or _in_provider_set("vertex_text_models", model)
|
||||
or model in litellm.models_by_provider["vertex_ai"]
|
||||
):
|
||||
if "VERTEXAI_PROJECT" in os.environ and "VERTEXAI_LOCATION" in os.environ:
|
||||
|
|
@ -6914,7 +6916,7 @@ def validate_environment(
|
|||
else:
|
||||
missing_keys.append("HUGGINGFACE_API_KEY")
|
||||
## ai21
|
||||
elif model in litellm.ai21_models:
|
||||
elif _in_provider_set("ai21_models", model):
|
||||
if "AI21_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
|
|
@ -6926,34 +6928,34 @@ def validate_environment(
|
|||
else:
|
||||
missing_keys.append("TOGETHERAI_API_KEY")
|
||||
## aleph_alpha
|
||||
elif model in litellm.aleph_alpha_models:
|
||||
elif _in_provider_set("aleph_alpha_models", model):
|
||||
if "ALEPH_ALPHA_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("ALEPH_ALPHA_API_KEY")
|
||||
## baseten
|
||||
elif model in litellm.baseten_models:
|
||||
elif _in_provider_set("baseten_models", model):
|
||||
if "BASETEN_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("BASETEN_API_KEY")
|
||||
## nlp_cloud
|
||||
elif model in litellm.nlp_cloud_models:
|
||||
elif _in_provider_set("nlp_cloud_models", model):
|
||||
if "NLP_CLOUD_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("NLP_CLOUD_API_KEY")
|
||||
elif model in litellm.novita_models:
|
||||
elif _in_provider_set("novita_models", model):
|
||||
if "NOVITA_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("NOVITA_API_KEY")
|
||||
elif model in litellm.nebius_models:
|
||||
elif _in_provider_set("nebius_models", model):
|
||||
if "NEBIUS_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
missing_keys.append("NEBIUS_API_KEY")
|
||||
elif model in litellm.wandb_models:
|
||||
elif _in_provider_set("wandb_models", model):
|
||||
if "WANDB_API_KEY" in os.environ:
|
||||
keys_in_environment = True
|
||||
else:
|
||||
|
|
@ -6985,7 +6987,9 @@ def acreate(*args, **kwargs): ## Thin client to handle the acreate langchain ca
|
|||
def valid_model(model):
|
||||
try:
|
||||
# for a given model name, check if the user has the right permissions to access the model
|
||||
if model in litellm.open_ai_chat_completion_models or model in litellm.open_ai_text_completion_models:
|
||||
if _in_provider_set("open_ai_chat_completion_models", model) or _in_provider_set(
|
||||
"open_ai_text_completion_models", model
|
||||
):
|
||||
openai.models.retrieve(model)
|
||||
else:
|
||||
messages: Final = [{"role": "user", "content": "Hello World"}]
|
||||
|
|
@ -8494,11 +8498,11 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return VertexAIGPTOSSTransformation()
|
||||
elif model in litellm.vertex_mistral_models:
|
||||
elif _in_provider_set("vertex_mistral_models", model):
|
||||
if "codestral" in model:
|
||||
return litellm.CodestralTextCompletionConfig()
|
||||
return litellm.VertexAIMistralConfig()
|
||||
elif model in litellm.vertex_ai_ai21_models:
|
||||
elif _in_provider_set("vertex_ai_ai21_models", model):
|
||||
return litellm.VertexAIAi21Config()
|
||||
else:
|
||||
return litellm.VertexAILlama3Config()
|
||||
|
|
|
|||
70
tests/unit/_catalog/test_provider_sets.py
Normal file
70
tests/unit/_catalog/test_provider_sets.py
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm._catalog import CATALOG
|
||||
from litellm._catalog.provider_sets import models_for
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_map(monkeypatch: pytest.MonkeyPatch) -> dict[str, dict[str, object]]:
|
||||
rows: Final = dict(CATALOG.bundled().served())
|
||||
monkeypatch.setattr(litellm, "model_cost", rows)
|
||||
return rows
|
||||
|
||||
|
||||
def test_a_row_added_dropped_or_moved_shows_on_the_next_read(live_map: dict[str, dict[str, object]]) -> None:
|
||||
"""provider-sets-live. On main a moved key stayed in its old set and a direct write reached none"""
|
||||
live_map["acme/new-chat"] = {"litellm_provider": "cohere", "mode": "chat"}
|
||||
assert "acme/new-chat" in litellm.cohere_models
|
||||
assert "acme/new-chat" in litellm.models_by_provider["cohere"]
|
||||
|
||||
live_map["acme/new-chat"] = {"litellm_provider": "mistral", "mode": "chat"}
|
||||
live_map["acme/len-change"] = {}
|
||||
assert "acme/new-chat" not in litellm.cohere_models
|
||||
assert "acme/new-chat" in litellm.mistral_chat_models
|
||||
|
||||
del live_map["acme/new-chat"]
|
||||
assert "acme/new-chat" not in litellm.models_by_provider["mistral"]
|
||||
|
||||
|
||||
def test_the_provider_sets_are_read_only() -> None:
|
||||
"""provider-sets-readonly"""
|
||||
with pytest.raises(AttributeError):
|
||||
litellm.anthropic_models.add("x") # pyright: ignore[reportAttributeAccessIssue] # the mutation under test
|
||||
with pytest.raises(TypeError):
|
||||
litellm.models_by_provider["openai"] = frozenset() # pyright: ignore[reportIndexIssue] # the mutation under test
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("live_map")
|
||||
def test_every_provider_in_the_map_lists_the_rows_that_name_it() -> None:
|
||||
providers: Final = frozenset(
|
||||
row["litellm_provider"] for row in litellm.model_cost.values() if isinstance(row.get("litellm_provider"), str)
|
||||
)
|
||||
for provider in providers:
|
||||
assert provider in litellm.models_by_provider
|
||||
assert "together_ai" in litellm.models_by_provider
|
||||
assert all(litellm.model_cost[key]["litellm_provider"] == "aihubmix" for key in models_for("aihubmix"))
|
||||
|
||||
|
||||
def test_registering_an_openrouter_model_adds_no_stripped_name(live_map: dict[str, dict[str, object]]) -> None:
|
||||
litellm.register_model({"openrouter/acme/model": {"litellm_provider": "openrouter", "mode": "chat"}})
|
||||
|
||||
assert "openrouter/acme/model" in litellm.openrouter_models
|
||||
assert "acme/model" not in litellm.openrouter_models
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("live_map")
|
||||
def test_the_request_path_membership_check_agrees_with_every_derived_set() -> None:
|
||||
from litellm._catalog.provider_sets import LEGACY_NAMES, contains, legacy_set
|
||||
|
||||
names: Final = frozenset(litellm.model_cost) | {key.replace("vertex_ai/", "") for key in litellm.model_cost}
|
||||
disagreements: Final = [
|
||||
(set_name, model)
|
||||
for set_name in LEGACY_NAMES
|
||||
for model in names # comprehension-ok: every set against every candidate name
|
||||
if contains(set_name, model) != (model in legacy_set(set_name))
|
||||
]
|
||||
|
||||
assert disagreements == []
|
||||
|
|
@ -44,18 +44,27 @@ os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
|||
import litellm # noqa: E402 # litellm reads LITELLM_LOCAL_MODEL_COST_MAP at import
|
||||
import litellm.router as litellm_router_module # noqa: E402 # same import-time dependency
|
||||
import litellm.utils as litellm_utils_module # noqa: E402 # same import-time dependency
|
||||
from litellm._catalog.provider_sets import LEGACY_NAMES as LEGACY_PROVIDER_SETS # noqa: E402 # after litellm
|
||||
from litellm._logging import ALL_LOGGERS # noqa: E402 # same import-time dependency
|
||||
from litellm.anthropic_beta_headers_manager import (
|
||||
reload_beta_headers_config, # noqa: E402 # same import-time dependency
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER # noqa: E402 # same import-time dependency
|
||||
from litellm.anthropic_beta_headers_manager import reload_beta_headers_config # noqa: E402 # same import-time dependency
|
||||
from litellm.litellm_core_utils.prompt_templates import factory as prompt_factory_module # noqa: E402 # same import-time dependency
|
||||
from litellm.litellm_core_utils.prompt_templates import (
|
||||
factory as prompt_factory_module, # noqa: E402 # same import-time dependency
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates import ( # noqa: E402 # same import-time dependency
|
||||
image_handling as image_handling_module,
|
||||
)
|
||||
from litellm.llms.gemini.chat import transformation as gemini_chat_transformation_module # noqa: E402 # same import-time dependency
|
||||
from litellm.llms.custom_httpx.async_client_cleanup import ( # noqa: E402 # same import-time dependency
|
||||
close_litellm_async_clients,
|
||||
)
|
||||
from litellm.proxy.db import tool_registry_writer as tool_registry_writer_module # noqa: E402 # same import-time dependency
|
||||
from litellm.llms.gemini.chat import (
|
||||
transformation as gemini_chat_transformation_module, # noqa: E402 # same import-time dependency
|
||||
)
|
||||
from litellm.proxy.db import (
|
||||
tool_registry_writer as tool_registry_writer_module, # noqa: E402 # same import-time dependency
|
||||
)
|
||||
|
||||
LOOPBACK_HOSTS: Final = ["127.0.0.1", "::1", "localhost"]
|
||||
AMBIENT_AZURE_CREDENTIAL_ENV_VARS: Final = (
|
||||
|
|
@ -124,8 +133,6 @@ RESTORED_GLOBALS: Final = (
|
|||
"cost_discount_config",
|
||||
"disable_hf_tokenizer_download",
|
||||
"disable_copilot_system_to_assistant",
|
||||
"cohere_models",
|
||||
"anthropic_models",
|
||||
"token_counter",
|
||||
"initialized_langfuse_clients",
|
||||
)
|
||||
|
|
@ -309,26 +316,15 @@ def isolate_router_model_cost_state() -> Iterator[None]:
|
|||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _provider_model_sets() -> tuple[set[str], ...]:
|
||||
module_globals: Final = tuple(vars(litellm).items())
|
||||
return tuple(models for name, models in module_globals if name.endswith("_models") and isinstance(models, set))
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate_model_catalog() -> Iterator[None]:
|
||||
model_cost: Final = litellm.model_cost
|
||||
model_cost_entries: Final = dict(model_cost)
|
||||
models_by_provider: Final = litellm.models_by_provider
|
||||
models_by_provider_entries: Final = dict(models_by_provider)
|
||||
provider_model_sets: Final = tuple((models, frozenset(models)) for models in _provider_model_sets())
|
||||
yield
|
||||
model_cost.clear()
|
||||
model_cost.update(model_cost_entries)
|
||||
models_by_provider.clear()
|
||||
models_by_provider.update(models_by_provider_entries)
|
||||
for models, entries in provider_model_sets:
|
||||
models.clear()
|
||||
models.update(entries)
|
||||
for name in LEGACY_PROVIDER_SETS:
|
||||
vars(litellm).pop(name, None)
|
||||
litellm_utils_module._invalidate_model_cost_lowercase_map()
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
|
|
|||
|
|
@ -264,10 +264,8 @@ def test_get_llm_provider_deepseek_custom_api_base():
|
|||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_llm_provider_vertex_ai_image_models(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "vertex_ai_image_models", set())
|
||||
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
|
||||
litellm.add_known_models(
|
||||
model_cost_map={
|
||||
litellm.model_cost.update(
|
||||
{
|
||||
"vertex_ai/imagegeneration@006": {
|
||||
"litellm_provider": "vertex_ai-image-models",
|
||||
"mode": "image_generation",
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
|
|||
import path fetches the published map, which lags a key added in this repo."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
litellm.model_cost.update(litellm.model_cost)
|
||||
|
||||
|
||||
def test_azure_gpt5_supports_reasoning_effort(config: AzureOpenAIGPT5Config):
|
||||
|
|
|
|||
|
|
@ -788,7 +788,7 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
|
|||
"""Pin the bundled cost map: the published map lags a key added in this repo."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
litellm.model_cost.update(litellm.model_cost)
|
||||
|
||||
|
||||
def test_azure_responses_reasoning_effort_none_unlocks_temperature_off_the_azure_row(
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm._catalog.provider_sets import OPAQUE_KEYS
|
||||
from litellm.llms.baseten.chat import BasetenConfig
|
||||
from litellm.utils import get_valid_models
|
||||
|
||||
|
|
@ -66,7 +67,7 @@ class TestBasetenModelDiscovery:
|
|||
key
|
||||
for key, value in litellm.model_cost.items()
|
||||
if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value)
|
||||
}
|
||||
} - OPAQUE_KEYS
|
||||
|
||||
assert servable_registry_models
|
||||
assert servable_registry_models <= set(baseten_models)
|
||||
|
|
@ -82,15 +83,14 @@ class TestBasetenModelDiscovery:
|
|||
"""A refreshed cost map that marks a listed model retired removes it without a restart."""
|
||||
model: Final = "baseten/acme/reload-probe"
|
||||
try:
|
||||
litellm.add_known_models({model: {"litellm_provider": "baseten"}})
|
||||
litellm.model_cost.update({model: {"litellm_provider": "baseten"}})
|
||||
assert model in litellm.models_by_provider["baseten"]
|
||||
|
||||
litellm.add_known_models({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}})
|
||||
litellm.model_cost.update({model: {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}})
|
||||
|
||||
assert model not in litellm.models_by_provider["baseten"]
|
||||
finally:
|
||||
litellm.baseten_models.discard(model)
|
||||
litellm.add_known_models({})
|
||||
litellm.model_cost.update({})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value, expected",
|
||||
|
|
|
|||
|
|
@ -21,21 +21,11 @@ from litellm.types.utils import (
|
|||
def local_model_cost_map(monkeypatch):
|
||||
"""Resolve models against this checkout's cost map instead of the network-fetched
|
||||
``main`` copy, which lags this branch until merge."""
|
||||
original_converse_models = set(litellm.bedrock_converse_models)
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
litellm.get_model_info.cache_clear()
|
||||
try:
|
||||
litellm.bedrock_converse_models.update(
|
||||
key
|
||||
for key, value in litellm.model_cost.items()
|
||||
if isinstance(value, dict) and value.get("litellm_provider") == "bedrock_converse"
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
litellm.bedrock_converse_models.clear()
|
||||
litellm.bedrock_converse_models.update(original_converse_models)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
class GptProfile(NamedTuple):
|
||||
|
|
|
|||
|
|
@ -29,17 +29,13 @@ from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe
|
|||
@pytest.fixture
|
||||
def local_cost_map(monkeypatch):
|
||||
original_model_cost = litellm.model_cost
|
||||
original_bedrock_mantle_models = set(litellm.bedrock_mantle_models)
|
||||
try:
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
litellm.get_model_info.cache_clear()
|
||||
litellm.add_known_models()
|
||||
yield
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
litellm.bedrock_mantle_models.clear()
|
||||
litellm.bedrock_mantle_models.update(original_bedrock_mantle_models)
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -164,9 +164,8 @@ class TestRegistration:
|
|||
monkeypatch.setenv("EDENAI_API_KEY", "eden-test-key")
|
||||
present = litellm.validate_environment(model=alias)
|
||||
finally:
|
||||
litellm.edenai_models.discard(alias)
|
||||
litellm.model_cost.pop(alias, None)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.update({})
|
||||
|
||||
assert (missing["keys_in_environment"], missing["missing_keys"]) == (False, ["EDENAI_API_KEY"])
|
||||
assert (present["keys_in_environment"], present["missing_keys"]) == (True, [])
|
||||
|
|
@ -178,12 +177,11 @@ class TestRegistration:
|
|||
wildcard = LiteLLM_Params(model="edenai/*", api_key="wildcard-key")
|
||||
assert alias not in (get_provider_models("edenai", wildcard) or [])
|
||||
|
||||
litellm.add_known_models(model_cost_map={alias: {"litellm_provider": "edenai", "mode": "chat"}})
|
||||
litellm.model_cost.update({alias: {"litellm_provider": "edenai", "mode": "chat"}})
|
||||
try:
|
||||
expanded = get_provider_models("edenai", wildcard)
|
||||
finally:
|
||||
litellm.edenai_models.discard(alias)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.pop(alias, None)
|
||||
|
||||
assert expanded is not None
|
||||
assert alias in expanded
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
|
|||
on remote catalog fetches (and don't change behavior across remote refreshes)."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=""))
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
litellm.model_cost.update(litellm.model_cost)
|
||||
|
||||
|
||||
class TestGithubCopilotResponsesAPITransformation:
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
|
|||
monkeypatch.setattr(
|
||||
litellm, "model_cost", get_model_cost_map(url="")
|
||||
)
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
litellm.model_cost.update(litellm.model_cost)
|
||||
|
||||
|
||||
def test_gpt5_supports_reasoning_effort(config: OpenAIConfig):
|
||||
|
|
|
|||
|
|
@ -743,7 +743,7 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
|
|||
litellm.model_cost but never re-ran add_known_models(), so the provider sets
|
||||
stayed stale and wildcard matching failed for new models.
|
||||
|
||||
Fix: each reload now calls litellm.add_known_models(model_cost_map=new_map)
|
||||
Fix: each reload now calls litellm.model_cost.update(new_map)
|
||||
with the fetched map passed explicitly to avoid any reference ambiguity.
|
||||
"""
|
||||
from litellm.proxy._types import ProxyException
|
||||
|
|
@ -762,11 +762,8 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
|
|||
original_model_cost = litellm.model_cost
|
||||
litellm.model_cost = new_cost_map
|
||||
|
||||
# Confirm the model is NOT yet in the provider set before reload propagation.
|
||||
assert model not in litellm.anthropic_models
|
||||
|
||||
# Simulate what proxy_server.py now does after every reload.
|
||||
litellm.add_known_models(model_cost_map=new_cost_map)
|
||||
litellm.model_cost.update(new_cost_map)
|
||||
|
||||
# After add_known_models(), the model must be in the set.
|
||||
assert model in litellm.anthropic_models
|
||||
|
|
@ -804,7 +801,6 @@ async def test_wildcard_access_after_cost_map_reload(key_models, model, expect_t
|
|||
)
|
||||
finally:
|
||||
litellm.model_cost = original_model_cost
|
||||
litellm.anthropic_models.discard(model)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -816,7 +812,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
|
|||
wildcard access checks (anthropic/*, openai/*, …) work immediately.
|
||||
|
||||
This covers the proxy_server.py fix where both reload paths now call
|
||||
litellm.add_known_models(model_cost_map=new_model_cost_map) instead of
|
||||
litellm.model_cost.update(new_model_cost_map) instead of
|
||||
relying on the module-level model_cost being up to date.
|
||||
"""
|
||||
fake_new_model = "claude-brand-new-explicit-map-test"
|
||||
|
|
@ -835,7 +831,7 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
|
|||
# Simulate what proxy_server.py does on reload.
|
||||
original_model_cost = litellm.model_cost
|
||||
litellm.model_cost = new_cost_map
|
||||
litellm.add_known_models(model_cost_map=new_cost_map)
|
||||
litellm.model_cost.update(new_cost_map)
|
||||
|
||||
try:
|
||||
assert fake_new_model in litellm.anthropic_models, (
|
||||
|
|
@ -845,7 +841,6 @@ async def test_add_known_models_explicit_map_updates_provider_sets():
|
|||
finally:
|
||||
# Clean up: restore original state.
|
||||
litellm.model_cost = original_model_cost
|
||||
litellm.anthropic_models.discard(fake_new_model)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -921,16 +921,13 @@ def test_add_known_models_refreshes_models_by_provider_for_wildcard_expansion():
|
|||
captured_reference = litellm.models_by_provider
|
||||
assert fake_model not in litellm.models_by_provider["vertex_ai"]
|
||||
try:
|
||||
litellm.add_known_models(
|
||||
model_cost_map={fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}}
|
||||
)
|
||||
litellm.model_cost.update({fake_model: {"litellm_provider": "vertex_ai-language-models", "mode": "chat"}})
|
||||
assert fake_model in litellm.models_by_provider["vertex_ai"]
|
||||
assert litellm.models_by_provider is captured_reference
|
||||
assert fake_model in captured_reference["vertex_ai"]
|
||||
assert fake_model in get_known_models_from_wildcard("vertex_ai/*")
|
||||
finally:
|
||||
litellm.vertex_language_models.discard(fake_model)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.pop(fake_model, None)
|
||||
assert fake_model not in litellm.models_by_provider["vertex_ai"]
|
||||
|
||||
|
||||
|
|
@ -949,9 +946,7 @@ def test_wildcard_lists_a_model_once_when_registered_both_bare_and_provider_pref
|
|||
finally:
|
||||
litellm.model_cost.pop(bare, None)
|
||||
litellm.model_cost.pop(f"openai/{bare}", None)
|
||||
litellm.open_ai_chat_completion_models.discard(bare)
|
||||
litellm.open_ai_chat_completion_models.discard(f"openai/{bare}")
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.update({})
|
||||
|
||||
|
||||
def test_get_complete_model_list_drops_no_default_models_sentinel():
|
||||
|
|
@ -1216,7 +1211,7 @@ def _registered_chat_models(provider_by_model: Mapping[str, str]) -> Generator[N
|
|||
litellm.model_cost.pop(model, None)
|
||||
for provider_models in provider_model_sets:
|
||||
provider_models.discard(model)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.update({})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
|
|
@ -9058,7 +9058,6 @@ def test_cost_calculator_with_custom_pricing():
|
|||
@pytest.fixture
|
||||
def isolated_pricing_registry(local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
monkeypatch.setattr(litellm, "open_ai_chat_completion_models", set(litellm.open_ai_chat_completion_models))
|
||||
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4314,17 +4314,14 @@ def _gemini_client_returning_a_reply():
|
|||
|
||||
@pytest.fixture
|
||||
def restore_model_registry():
|
||||
"""litellm.model_cost and the provider name sets are module-global.
|
||||
"""litellm.model_cost is module-global, and the provider sets are derived from it.
|
||||
|
||||
register_model merges into the existing entry in place, hence the deep copy.
|
||||
"""
|
||||
model_cost = copy.deepcopy(litellm.model_cost)
|
||||
openai_models = set(litellm.open_ai_chat_completion_models)
|
||||
yield
|
||||
litellm.model_cost.clear()
|
||||
litellm.model_cost.update(model_cost)
|
||||
litellm.open_ai_chat_completion_models.clear()
|
||||
litellm.open_ai_chat_completion_models.update(openai_models)
|
||||
|
||||
|
||||
def test_openai_model_name_does_not_outrank_explicit_provider():
|
||||
|
|
@ -4367,7 +4364,6 @@ def test_mislabelled_pricing_entry_does_not_reroute_provider(restore_model_regis
|
|||
}
|
||||
}
|
||||
)
|
||||
assert "gemini-2.5-pro" in litellm.open_ai_chat_completion_models
|
||||
client, post = _gemini_client_returning_a_reply()
|
||||
|
||||
with patch.object(client, "post", new=post):
|
||||
|
|
|
|||
|
|
@ -8376,7 +8376,7 @@ def test_get_deployment_model_info_base_model_flow():
|
|||
print("✓ All base model flow test cases passed!")
|
||||
|
||||
|
||||
@patch("litellm.model_cost", {})
|
||||
@patch("litellm.model_cost", {"gpt-3.5-turbo": {"litellm_provider": "openai", "mode": "chat"}})
|
||||
def test_get_deployment_model_info_base_model_merge_priority():
|
||||
"""Test that base model info merging respects the correct priority order"""
|
||||
from unittest.mock import patch
|
||||
|
|
|
|||
|
|
@ -3819,7 +3819,7 @@ def _simulate_price_data_reload_with_provider_sets(monkeypatch, fetched_catalog)
|
|||
`reload_model_cost_map` does before replaying, so bare names in the new catalog resolve."""
|
||||
monkeypatch.setattr(litellm, "model_cost", fetched_catalog)
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
litellm.add_known_models(model_cost_map=fetched_catalog)
|
||||
litellm.model_cost.update(fetched_catalog)
|
||||
CATALOG.replay_registrations()
|
||||
|
||||
|
||||
|
|
@ -3859,8 +3859,7 @@ def test_a_config_deployment_dropped_by_a_stale_cost_map_comes_back_on_reload(mo
|
|||
assert [d["model_info"]["id"] for d in router.model_list] == ["control-id", "new-id"]
|
||||
assert "new-id" in litellm.model_cost
|
||||
finally:
|
||||
litellm.open_ai_chat_completion_models.discard(backend)
|
||||
litellm.models_by_provider["openai"].discard(backend)
|
||||
pass
|
||||
|
||||
|
||||
def test_a_config_deployment_dropped_for_a_permanent_reason_is_not_retried_on_reload(monkeypatch):
|
||||
|
|
|
|||
|
|
@ -2299,10 +2299,8 @@ def test_add_known_models_skips_region_scoped_bedrock_pricing_keys(monkeypatch:
|
|||
"bedrock/us-west-1/migration.bedrock-model-v1:0": {**price, "litellm_provider": "bedrock"},
|
||||
"migration-openai-model": {**price, "litellm_provider": "openai"},
|
||||
}
|
||||
monkeypatch.setattr(litellm, "bedrock_models", set(litellm.bedrock_models))
|
||||
monkeypatch.setattr(litellm, "models_by_provider", dict(litellm.models_by_provider))
|
||||
|
||||
litellm.add_known_models(model_cost_map=model_cost_map)
|
||||
litellm.model_cost.update(model_cost_map)
|
||||
|
||||
assert "migration.bedrock-model-v1:0" in litellm.bedrock_models
|
||||
assert "bedrock/us-west-1/migration.bedrock-model-v1:0" not in litellm.bedrock_models
|
||||
|
|
@ -2416,9 +2414,6 @@ def test_register_model_openrouter_without_slash():
|
|||
always works.
|
||||
"""
|
||||
# Clear any existing entries
|
||||
litellm.openrouter_models.discard("my-custom-alias")
|
||||
litellm.openrouter_models.discard("gpt-4")
|
||||
litellm.openrouter_models.discard("openai/gpt-4")
|
||||
|
||||
# Test 1: Model name without '/' (this was the bug - would raise IndexError)
|
||||
litellm.register_model(
|
||||
|
|
@ -2446,7 +2441,8 @@ def test_register_model_openrouter_without_slash():
|
|||
},
|
||||
}
|
||||
)
|
||||
assert "gpt-4" in litellm.openrouter_models
|
||||
assert "openrouter/gpt-4" in litellm.openrouter_models
|
||||
assert "gpt-4" not in litellm.openrouter_models
|
||||
|
||||
# Test 3: Model name with double '/' (openrouter/provider/model format)
|
||||
litellm.register_model(
|
||||
|
|
@ -2460,7 +2456,8 @@ def test_register_model_openrouter_without_slash():
|
|||
},
|
||||
}
|
||||
)
|
||||
assert "openai/gpt-4-turbo" in litellm.openrouter_models
|
||||
assert "openrouter/openai/gpt-4-turbo" in litellm.openrouter_models
|
||||
assert "openai/gpt-4-turbo" not in litellm.openrouter_models
|
||||
|
||||
|
||||
def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_a_restart():
|
||||
|
|
@ -2484,8 +2481,6 @@ def test_register_model_makes_the_model_resolvable_and_wildcard_visible_without_
|
|||
assert model_name in litellm.models_by_provider["vertex_ai"]
|
||||
finally:
|
||||
litellm.model_cost.pop(model_name, None)
|
||||
litellm.vertex_language_models.discard(model_name)
|
||||
litellm.models_by_provider["vertex_ai"].discard(model_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -2511,8 +2506,7 @@ def test_registered_chat_model_lists_under_its_provider_wildcard(
|
|||
assert f"{wildcard.removesuffix('*')}{model_name}" in listed
|
||||
finally:
|
||||
litellm.model_cost.pop(model_name, None)
|
||||
getattr(litellm, provider_set).discard(model_name)
|
||||
litellm.add_known_models(model_cost_map={})
|
||||
litellm.model_cost.update({})
|
||||
|
||||
|
||||
def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing():
|
||||
|
|
@ -5008,17 +5002,10 @@ def test_model_cost_walks_survive_a_registration_landing_mid_walk(monkeypatch: p
|
|||
litellm.model_cost.setdefault("lit-5853-registered-during-the-rebuild", {"mode": "chat"})
|
||||
return super().lower()
|
||||
|
||||
class _EntryRegisteredMidWalk(dict):
|
||||
def get(self, field: str, default: object = None) -> object:
|
||||
litellm.model_cost.setdefault("lit-5853-registered-during-add-known-models", {"mode": "chat"})
|
||||
return super().get(field, default)
|
||||
|
||||
monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost))
|
||||
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = _EntryRegisteredMidWalk(mode="chat")
|
||||
litellm.model_cost[_KeyRegisteredMidWalk("LIT-5853-WALKED-LAST")] = {"mode": "chat"}
|
||||
try:
|
||||
assert get_model_cost_key("lit-5853-walked-last") == "LIT-5853-WALKED-LAST"
|
||||
litellm.add_known_models(model_cost_map=litellm.model_cost)
|
||||
assert "lit-5853-registered-during-add-known-models" in litellm.model_cost
|
||||
finally:
|
||||
_invalidate_model_cost_lowercase_map()
|
||||
|
||||
|
|
@ -8811,17 +8798,8 @@ def reset_mock_cache() -> None:
|
|||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_openai_model_sets(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"open_ai_chat_completion_models",
|
||||
set(litellm.open_ai_chat_completion_models),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"open_ai_text_completion_models",
|
||||
set(litellm.open_ai_text_completion_models),
|
||||
)
|
||||
def isolated_openai_model_sets() -> None:
|
||||
"""The provider sets are derived from litellm.model_cost, which the autouse isolation restores"""
|
||||
|
||||
|
||||
def test_aget_valid_models():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue