mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
refactor(cost): dispatch providers and call types through mirrored enums
LlmProviders and CallTypes are generated from Python's enums in litellm/types/utils.py, pinned value-for-value against executed Python via the fixture pinning surface, and the catalog's token-cost dispatch now matches on them instead of strings: the pre-else provider set, the together and fireworks fallbacks, and the embedding call routing all go through the enum, with uses_inclusive_token_thresholds parsing XAI the same way. The _python_-shaped variant names keep Python's member spelling so the correspondence is mechanical
This commit is contained in:
parent
0f4ce30103
commit
26a5dcb96d
8 changed files with 1453 additions and 71 deletions
|
|
@ -280,5 +280,8 @@ pub fn get_token_base_cost(
|
|||
}
|
||||
|
||||
pub fn uses_inclusive_token_thresholds(provider: Option<&str>) -> bool {
|
||||
provider == Some("xai")
|
||||
provider.is_some_and(|name| {
|
||||
crate::provider::LlmProviders::parse(name)
|
||||
.is_some_and(|provider| provider == crate::provider::LlmProviders::XAI)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
493
litellm-rust/crates/cost/src/call_type.rs
Normal file
493
litellm-rust/crates/cost/src/call_type.rs
Normal file
|
|
@ -0,0 +1,493 @@
|
|||
//! Mirrors Python's CallTypes (litellm/types/utils.py), generated from its variant list.
|
||||
|
||||
// variant names mirror Python's enum members verbatim
|
||||
#![allow(non_camel_case_types)]
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum CallTypes {
|
||||
embedding,
|
||||
aembedding,
|
||||
completion,
|
||||
acompletion,
|
||||
atext_completion,
|
||||
text_completion,
|
||||
image_generation,
|
||||
aimage_generation,
|
||||
image_edit,
|
||||
aimage_edit,
|
||||
moderation,
|
||||
amoderation,
|
||||
atranscription,
|
||||
transcription,
|
||||
aspeech,
|
||||
speech,
|
||||
rerank,
|
||||
arerank,
|
||||
search,
|
||||
asearch,
|
||||
arealtime,
|
||||
aresponses_websocket,
|
||||
create_batch,
|
||||
acreate_batch,
|
||||
aretrieve_batch,
|
||||
retrieve_batch,
|
||||
acancel_batch,
|
||||
cancel_batch,
|
||||
pass_through,
|
||||
anthropic_messages,
|
||||
aanthropic_messages,
|
||||
get_assistants,
|
||||
aget_assistants,
|
||||
create_assistants,
|
||||
acreate_assistants,
|
||||
delete_assistant,
|
||||
adelete_assistant,
|
||||
acreate_thread,
|
||||
create_thread,
|
||||
aget_thread,
|
||||
get_thread,
|
||||
a_add_message,
|
||||
add_message,
|
||||
aget_messages,
|
||||
get_messages,
|
||||
arun_thread,
|
||||
run_thread,
|
||||
arun_thread_stream,
|
||||
run_thread_stream,
|
||||
afile_retrieve,
|
||||
file_retrieve,
|
||||
afile_delete,
|
||||
file_delete,
|
||||
afile_list,
|
||||
file_list,
|
||||
acreate_file,
|
||||
create_file,
|
||||
afile_content,
|
||||
file_content,
|
||||
create_fine_tuning_job,
|
||||
acreate_fine_tuning_job,
|
||||
create_video,
|
||||
acreate_video,
|
||||
video_generation,
|
||||
avideo_generation,
|
||||
avideo_retrieve,
|
||||
video_retrieve,
|
||||
avideo_content,
|
||||
video_content,
|
||||
video_remix,
|
||||
avideo_remix,
|
||||
video_list,
|
||||
avideo_list,
|
||||
video_retrieve_job,
|
||||
avideo_retrieve_job,
|
||||
video_delete,
|
||||
avideo_delete,
|
||||
video_create_character,
|
||||
avideo_create_character,
|
||||
video_get_character,
|
||||
avideo_get_character,
|
||||
video_edit,
|
||||
avideo_edit,
|
||||
video_extension,
|
||||
avideo_extension,
|
||||
vector_store_file_create,
|
||||
avector_store_file_create,
|
||||
vector_store_file_list,
|
||||
avector_store_file_list,
|
||||
vector_store_file_retrieve,
|
||||
avector_store_file_retrieve,
|
||||
vector_store_file_content,
|
||||
avector_store_file_content,
|
||||
vector_store_file_update,
|
||||
avector_store_file_update,
|
||||
vector_store_file_delete,
|
||||
avector_store_file_delete,
|
||||
vector_store_create,
|
||||
avector_store_create,
|
||||
vector_store_search,
|
||||
avector_store_search,
|
||||
ingest,
|
||||
aingest,
|
||||
query,
|
||||
aquery,
|
||||
create_interaction,
|
||||
acreate_interaction,
|
||||
create_container,
|
||||
acreate_container,
|
||||
list_containers,
|
||||
alist_containers,
|
||||
retrieve_container,
|
||||
aretrieve_container,
|
||||
delete_container,
|
||||
adelete_container,
|
||||
list_container_files,
|
||||
alist_container_files,
|
||||
upload_container_file,
|
||||
aupload_container_file,
|
||||
create_sandbox,
|
||||
acreate_sandbox,
|
||||
delete_sandbox,
|
||||
adelete_sandbox,
|
||||
run_code,
|
||||
arun_code,
|
||||
code_interpreter_tool,
|
||||
acode_interpreter_tool,
|
||||
acancel_fine_tuning_job,
|
||||
cancel_fine_tuning_job,
|
||||
alist_fine_tuning_jobs,
|
||||
list_fine_tuning_jobs,
|
||||
aretrieve_fine_tuning_job,
|
||||
retrieve_fine_tuning_job,
|
||||
responses,
|
||||
aresponses,
|
||||
alist_input_items,
|
||||
llm_passthrough_route,
|
||||
allm_passthrough_route,
|
||||
generate_content,
|
||||
agenerate_content,
|
||||
generate_content_stream,
|
||||
agenerate_content_stream,
|
||||
ocr,
|
||||
aocr,
|
||||
call_mcp_tool,
|
||||
list_mcp_tools,
|
||||
asend_message,
|
||||
send_message,
|
||||
acreate_skill,
|
||||
}
|
||||
|
||||
impl CallTypes {
|
||||
pub const VALUES: [Self; 149] = [
|
||||
Self::embedding,
|
||||
Self::aembedding,
|
||||
Self::completion,
|
||||
Self::acompletion,
|
||||
Self::atext_completion,
|
||||
Self::text_completion,
|
||||
Self::image_generation,
|
||||
Self::aimage_generation,
|
||||
Self::image_edit,
|
||||
Self::aimage_edit,
|
||||
Self::moderation,
|
||||
Self::amoderation,
|
||||
Self::atranscription,
|
||||
Self::transcription,
|
||||
Self::aspeech,
|
||||
Self::speech,
|
||||
Self::rerank,
|
||||
Self::arerank,
|
||||
Self::search,
|
||||
Self::asearch,
|
||||
Self::arealtime,
|
||||
Self::aresponses_websocket,
|
||||
Self::create_batch,
|
||||
Self::acreate_batch,
|
||||
Self::aretrieve_batch,
|
||||
Self::retrieve_batch,
|
||||
Self::acancel_batch,
|
||||
Self::cancel_batch,
|
||||
Self::pass_through,
|
||||
Self::anthropic_messages,
|
||||
Self::aanthropic_messages,
|
||||
Self::get_assistants,
|
||||
Self::aget_assistants,
|
||||
Self::create_assistants,
|
||||
Self::acreate_assistants,
|
||||
Self::delete_assistant,
|
||||
Self::adelete_assistant,
|
||||
Self::acreate_thread,
|
||||
Self::create_thread,
|
||||
Self::aget_thread,
|
||||
Self::get_thread,
|
||||
Self::a_add_message,
|
||||
Self::add_message,
|
||||
Self::aget_messages,
|
||||
Self::get_messages,
|
||||
Self::arun_thread,
|
||||
Self::run_thread,
|
||||
Self::arun_thread_stream,
|
||||
Self::run_thread_stream,
|
||||
Self::afile_retrieve,
|
||||
Self::file_retrieve,
|
||||
Self::afile_delete,
|
||||
Self::file_delete,
|
||||
Self::afile_list,
|
||||
Self::file_list,
|
||||
Self::acreate_file,
|
||||
Self::create_file,
|
||||
Self::afile_content,
|
||||
Self::file_content,
|
||||
Self::create_fine_tuning_job,
|
||||
Self::acreate_fine_tuning_job,
|
||||
Self::create_video,
|
||||
Self::acreate_video,
|
||||
Self::video_generation,
|
||||
Self::avideo_generation,
|
||||
Self::avideo_retrieve,
|
||||
Self::video_retrieve,
|
||||
Self::avideo_content,
|
||||
Self::video_content,
|
||||
Self::video_remix,
|
||||
Self::avideo_remix,
|
||||
Self::video_list,
|
||||
Self::avideo_list,
|
||||
Self::video_retrieve_job,
|
||||
Self::avideo_retrieve_job,
|
||||
Self::video_delete,
|
||||
Self::avideo_delete,
|
||||
Self::video_create_character,
|
||||
Self::avideo_create_character,
|
||||
Self::video_get_character,
|
||||
Self::avideo_get_character,
|
||||
Self::video_edit,
|
||||
Self::avideo_edit,
|
||||
Self::video_extension,
|
||||
Self::avideo_extension,
|
||||
Self::vector_store_file_create,
|
||||
Self::avector_store_file_create,
|
||||
Self::vector_store_file_list,
|
||||
Self::avector_store_file_list,
|
||||
Self::vector_store_file_retrieve,
|
||||
Self::avector_store_file_retrieve,
|
||||
Self::vector_store_file_content,
|
||||
Self::avector_store_file_content,
|
||||
Self::vector_store_file_update,
|
||||
Self::avector_store_file_update,
|
||||
Self::vector_store_file_delete,
|
||||
Self::avector_store_file_delete,
|
||||
Self::vector_store_create,
|
||||
Self::avector_store_create,
|
||||
Self::vector_store_search,
|
||||
Self::avector_store_search,
|
||||
Self::ingest,
|
||||
Self::aingest,
|
||||
Self::query,
|
||||
Self::aquery,
|
||||
Self::create_interaction,
|
||||
Self::acreate_interaction,
|
||||
Self::create_container,
|
||||
Self::acreate_container,
|
||||
Self::list_containers,
|
||||
Self::alist_containers,
|
||||
Self::retrieve_container,
|
||||
Self::aretrieve_container,
|
||||
Self::delete_container,
|
||||
Self::adelete_container,
|
||||
Self::list_container_files,
|
||||
Self::alist_container_files,
|
||||
Self::upload_container_file,
|
||||
Self::aupload_container_file,
|
||||
Self::create_sandbox,
|
||||
Self::acreate_sandbox,
|
||||
Self::delete_sandbox,
|
||||
Self::adelete_sandbox,
|
||||
Self::run_code,
|
||||
Self::arun_code,
|
||||
Self::code_interpreter_tool,
|
||||
Self::acode_interpreter_tool,
|
||||
Self::acancel_fine_tuning_job,
|
||||
Self::cancel_fine_tuning_job,
|
||||
Self::alist_fine_tuning_jobs,
|
||||
Self::list_fine_tuning_jobs,
|
||||
Self::aretrieve_fine_tuning_job,
|
||||
Self::retrieve_fine_tuning_job,
|
||||
Self::responses,
|
||||
Self::aresponses,
|
||||
Self::alist_input_items,
|
||||
Self::llm_passthrough_route,
|
||||
Self::allm_passthrough_route,
|
||||
Self::generate_content,
|
||||
Self::agenerate_content,
|
||||
Self::generate_content_stream,
|
||||
Self::agenerate_content_stream,
|
||||
Self::ocr,
|
||||
Self::aocr,
|
||||
Self::call_mcp_tool,
|
||||
Self::list_mcp_tools,
|
||||
Self::asend_message,
|
||||
Self::send_message,
|
||||
Self::acreate_skill,
|
||||
];
|
||||
|
||||
const NAMES: [(Self, &'static str); 149] = [
|
||||
(Self::embedding, "embedding"),
|
||||
(Self::aembedding, "aembedding"),
|
||||
(Self::completion, "completion"),
|
||||
(Self::acompletion, "acompletion"),
|
||||
(Self::atext_completion, "atext_completion"),
|
||||
(Self::text_completion, "text_completion"),
|
||||
(Self::image_generation, "image_generation"),
|
||||
(Self::aimage_generation, "aimage_generation"),
|
||||
(Self::image_edit, "image_edit"),
|
||||
(Self::aimage_edit, "aimage_edit"),
|
||||
(Self::moderation, "moderation"),
|
||||
(Self::amoderation, "amoderation"),
|
||||
(Self::atranscription, "atranscription"),
|
||||
(Self::transcription, "transcription"),
|
||||
(Self::aspeech, "aspeech"),
|
||||
(Self::speech, "speech"),
|
||||
(Self::rerank, "rerank"),
|
||||
(Self::arerank, "arerank"),
|
||||
(Self::search, "search"),
|
||||
(Self::asearch, "asearch"),
|
||||
(Self::arealtime, "_arealtime"),
|
||||
(Self::aresponses_websocket, "_aresponses_websocket"),
|
||||
(Self::create_batch, "create_batch"),
|
||||
(Self::acreate_batch, "acreate_batch"),
|
||||
(Self::aretrieve_batch, "aretrieve_batch"),
|
||||
(Self::retrieve_batch, "retrieve_batch"),
|
||||
(Self::acancel_batch, "acancel_batch"),
|
||||
(Self::cancel_batch, "cancel_batch"),
|
||||
(Self::pass_through, "pass_through_endpoint"),
|
||||
(Self::anthropic_messages, "anthropic_messages"),
|
||||
(Self::aanthropic_messages, "aanthropic_messages"),
|
||||
(Self::get_assistants, "get_assistants"),
|
||||
(Self::aget_assistants, "aget_assistants"),
|
||||
(Self::create_assistants, "create_assistants"),
|
||||
(Self::acreate_assistants, "acreate_assistants"),
|
||||
(Self::delete_assistant, "delete_assistant"),
|
||||
(Self::adelete_assistant, "adelete_assistant"),
|
||||
(Self::acreate_thread, "acreate_thread"),
|
||||
(Self::create_thread, "create_thread"),
|
||||
(Self::aget_thread, "aget_thread"),
|
||||
(Self::get_thread, "get_thread"),
|
||||
(Self::a_add_message, "a_add_message"),
|
||||
(Self::add_message, "add_message"),
|
||||
(Self::aget_messages, "aget_messages"),
|
||||
(Self::get_messages, "get_messages"),
|
||||
(Self::arun_thread, "arun_thread"),
|
||||
(Self::run_thread, "run_thread"),
|
||||
(Self::arun_thread_stream, "arun_thread_stream"),
|
||||
(Self::run_thread_stream, "run_thread_stream"),
|
||||
(Self::afile_retrieve, "afile_retrieve"),
|
||||
(Self::file_retrieve, "file_retrieve"),
|
||||
(Self::afile_delete, "afile_delete"),
|
||||
(Self::file_delete, "file_delete"),
|
||||
(Self::afile_list, "afile_list"),
|
||||
(Self::file_list, "file_list"),
|
||||
(Self::acreate_file, "acreate_file"),
|
||||
(Self::create_file, "create_file"),
|
||||
(Self::afile_content, "afile_content"),
|
||||
(Self::file_content, "file_content"),
|
||||
(Self::create_fine_tuning_job, "create_fine_tuning_job"),
|
||||
(Self::acreate_fine_tuning_job, "acreate_fine_tuning_job"),
|
||||
(Self::create_video, "create_video"),
|
||||
(Self::acreate_video, "acreate_video"),
|
||||
(Self::video_generation, "video_generation"),
|
||||
(Self::avideo_generation, "avideo_generation"),
|
||||
(Self::avideo_retrieve, "avideo_retrieve"),
|
||||
(Self::video_retrieve, "video_retrieve"),
|
||||
(Self::avideo_content, "avideo_content"),
|
||||
(Self::video_content, "video_content"),
|
||||
(Self::video_remix, "video_remix"),
|
||||
(Self::avideo_remix, "avideo_remix"),
|
||||
(Self::video_list, "video_list"),
|
||||
(Self::avideo_list, "avideo_list"),
|
||||
(Self::video_retrieve_job, "video_retrieve_job"),
|
||||
(Self::avideo_retrieve_job, "avideo_retrieve_job"),
|
||||
(Self::video_delete, "video_delete"),
|
||||
(Self::avideo_delete, "avideo_delete"),
|
||||
(Self::video_create_character, "video_create_character"),
|
||||
(Self::avideo_create_character, "avideo_create_character"),
|
||||
(Self::video_get_character, "video_get_character"),
|
||||
(Self::avideo_get_character, "avideo_get_character"),
|
||||
(Self::video_edit, "video_edit"),
|
||||
(Self::avideo_edit, "avideo_edit"),
|
||||
(Self::video_extension, "video_extension"),
|
||||
(Self::avideo_extension, "avideo_extension"),
|
||||
(Self::vector_store_file_create, "vector_store_file_create"),
|
||||
(Self::avector_store_file_create, "avector_store_file_create"),
|
||||
(Self::vector_store_file_list, "vector_store_file_list"),
|
||||
(Self::avector_store_file_list, "avector_store_file_list"),
|
||||
(
|
||||
Self::vector_store_file_retrieve,
|
||||
"vector_store_file_retrieve",
|
||||
),
|
||||
(
|
||||
Self::avector_store_file_retrieve,
|
||||
"avector_store_file_retrieve",
|
||||
),
|
||||
(Self::vector_store_file_content, "vector_store_file_content"),
|
||||
(
|
||||
Self::avector_store_file_content,
|
||||
"avector_store_file_content",
|
||||
),
|
||||
(Self::vector_store_file_update, "vector_store_file_update"),
|
||||
(Self::avector_store_file_update, "avector_store_file_update"),
|
||||
(Self::vector_store_file_delete, "vector_store_file_delete"),
|
||||
(Self::avector_store_file_delete, "avector_store_file_delete"),
|
||||
(Self::vector_store_create, "vector_store_create"),
|
||||
(Self::avector_store_create, "avector_store_create"),
|
||||
(Self::vector_store_search, "vector_store_search"),
|
||||
(Self::avector_store_search, "avector_store_search"),
|
||||
(Self::ingest, "ingest"),
|
||||
(Self::aingest, "aingest"),
|
||||
(Self::query, "query"),
|
||||
(Self::aquery, "aquery"),
|
||||
(Self::create_interaction, "create_interaction"),
|
||||
(Self::acreate_interaction, "acreate_interaction"),
|
||||
(Self::create_container, "create_container"),
|
||||
(Self::acreate_container, "acreate_container"),
|
||||
(Self::list_containers, "list_containers"),
|
||||
(Self::alist_containers, "alist_containers"),
|
||||
(Self::retrieve_container, "retrieve_container"),
|
||||
(Self::aretrieve_container, "aretrieve_container"),
|
||||
(Self::delete_container, "delete_container"),
|
||||
(Self::adelete_container, "adelete_container"),
|
||||
(Self::list_container_files, "list_container_files"),
|
||||
(Self::alist_container_files, "alist_container_files"),
|
||||
(Self::upload_container_file, "upload_container_file"),
|
||||
(Self::aupload_container_file, "aupload_container_file"),
|
||||
(Self::create_sandbox, "create_sandbox"),
|
||||
(Self::acreate_sandbox, "acreate_sandbox"),
|
||||
(Self::delete_sandbox, "delete_sandbox"),
|
||||
(Self::adelete_sandbox, "adelete_sandbox"),
|
||||
(Self::run_code, "run_code"),
|
||||
(Self::arun_code, "arun_code"),
|
||||
(Self::code_interpreter_tool, "code_interpreter_tool"),
|
||||
(Self::acode_interpreter_tool, "acode_interpreter_tool"),
|
||||
(Self::acancel_fine_tuning_job, "acancel_fine_tuning_job"),
|
||||
(Self::cancel_fine_tuning_job, "cancel_fine_tuning_job"),
|
||||
(Self::alist_fine_tuning_jobs, "alist_fine_tuning_jobs"),
|
||||
(Self::list_fine_tuning_jobs, "list_fine_tuning_jobs"),
|
||||
(Self::aretrieve_fine_tuning_job, "aretrieve_fine_tuning_job"),
|
||||
(Self::retrieve_fine_tuning_job, "retrieve_fine_tuning_job"),
|
||||
(Self::responses, "responses"),
|
||||
(Self::aresponses, "aresponses"),
|
||||
(Self::alist_input_items, "alist_input_items"),
|
||||
(Self::llm_passthrough_route, "llm_passthrough_route"),
|
||||
(Self::allm_passthrough_route, "allm_passthrough_route"),
|
||||
(Self::generate_content, "generate_content"),
|
||||
(Self::agenerate_content, "agenerate_content"),
|
||||
(Self::generate_content_stream, "generate_content_stream"),
|
||||
(Self::agenerate_content_stream, "agenerate_content_stream"),
|
||||
(Self::ocr, "ocr"),
|
||||
(Self::aocr, "aocr"),
|
||||
(Self::call_mcp_tool, "call_mcp_tool"),
|
||||
(Self::list_mcp_tools, "list_mcp_tools"),
|
||||
(Self::asend_message, "asend_message"),
|
||||
(Self::send_message, "send_message"),
|
||||
(Self::acreate_skill, "acreate_skill"),
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
Self::NAMES
|
||||
.iter()
|
||||
.find(|(variant, _)| *variant == self)
|
||||
.map(|(_, name)| *name)
|
||||
.expect("every variant carries its name")
|
||||
}
|
||||
|
||||
pub fn parse(name: &str) -> Option<Self> {
|
||||
Self::NAMES
|
||||
.iter()
|
||||
.find(|(_, known)| *known == name)
|
||||
.map(|(variant, _)| *variant)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CallTypes {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
|
@ -56,6 +56,7 @@ use crate::perplexity_cost::cost_per_token as perplexity_cost_per_token;
|
|||
use crate::prompt_caching_savings::{
|
||||
PromptCachingSavingsRequest, calculate_prompt_caching_savings,
|
||||
};
|
||||
use crate::provider::LlmProviders;
|
||||
use crate::provider_cache::apply_provider_cache_read_default;
|
||||
use crate::realtime_cost::{
|
||||
combine_usage_objects, event_usage, get_transcription_model_name_from_results,
|
||||
|
|
@ -473,6 +474,7 @@ impl ModelInfoCatalog {
|
|||
request: ModelCostRequest<'_>,
|
||||
call: CostCall<'_>,
|
||||
) -> Result<(f64, f64), CatalogCallError> {
|
||||
let provider = request.provider.and_then(LlmProviders::parse);
|
||||
match call {
|
||||
CostCall::Token {
|
||||
call_type,
|
||||
|
|
@ -480,7 +482,7 @@ impl ModelInfoCatalog {
|
|||
completion_characters,
|
||||
request_model,
|
||||
} => {
|
||||
if request.provider == Some("vertex_ai") {
|
||||
if provider == Some(LlmProviders::VERTEX_AI) {
|
||||
return Ok(self.vertex_cost(
|
||||
request,
|
||||
call_type,
|
||||
|
|
@ -488,12 +490,16 @@ impl ModelInfoCatalog {
|
|||
completion_characters,
|
||||
)?);
|
||||
}
|
||||
if request.provider == Some("together_ai")
|
||||
&& matches!(call_type, "embedding" | "aembedding")
|
||||
if provider == Some(LlmProviders::TOGETHER_AI)
|
||||
&& matches!(
|
||||
crate::call_type::CallTypes::parse(call_type),
|
||||
Some(crate::call_type::CallTypes::embedding)
|
||||
| Some(crate::call_type::CallTypes::aembedding)
|
||||
)
|
||||
{
|
||||
return Ok(self.together_ai_cost_per_token(request, call_type)?);
|
||||
}
|
||||
if request.provider == Some("azure_ai") {
|
||||
if provider == Some(LlmProviders::AZURE_AI) {
|
||||
return Ok(self.azure_ai_cost_per_token(request, request_model)?);
|
||||
}
|
||||
Ok(self.cost_per_token(request)?)
|
||||
|
|
@ -848,26 +854,27 @@ impl ModelInfoCatalog {
|
|||
{
|
||||
return Ok(cost);
|
||||
}
|
||||
if request.provider == Some("azure_ai") {
|
||||
let provider = request.provider.and_then(LlmProviders::parse);
|
||||
if provider == Some(LlmProviders::AZURE_AI) {
|
||||
return self.azure_ai_cost_per_token(request, None);
|
||||
}
|
||||
if request.provider == Some("databricks") {
|
||||
if provider == Some(LlmProviders::DATABRICKS) {
|
||||
return self.databricks_cost_per_token(request);
|
||||
}
|
||||
if request.provider == Some("lemonade") {
|
||||
if provider == Some(LlmProviders::LEMONADE) {
|
||||
return Ok(self.lemonade_cost_per_token(request));
|
||||
}
|
||||
if request.provider == Some("perplexity")
|
||||
if provider == Some(LlmProviders::PERPLEXITY)
|
||||
&& let Some(cost) = request.usage.cost
|
||||
{
|
||||
return Ok((0.0, cost));
|
||||
}
|
||||
if request.provider == Some("xai")
|
||||
if provider == Some(LlmProviders::XAI)
|
||||
&& let Some(cost) = xai_reported_cost(request.usage)
|
||||
{
|
||||
return Ok((0.0, cost));
|
||||
}
|
||||
let needs_together_fallback = (request.provider == Some("together_ai")
|
||||
let needs_together_fallback = (provider == Some(LlmProviders::TOGETHER_AI)
|
||||
|| request.model.contains("togethercomputer")
|
||||
|| request.model.contains("together_ai"))
|
||||
&& !has_together_registry_pricing(request.model, &self.entries);
|
||||
|
|
@ -894,70 +901,72 @@ impl ModelInfoCatalog {
|
|||
if let Some(cost) = per_second_pricing_cost(&model_info, request.response_time_ms) {
|
||||
return Ok(cost);
|
||||
}
|
||||
if request.provider == Some("azure")
|
||||
if provider == Some(LlmProviders::AZURE)
|
||||
&& let Some(cost) = output_per_second_cost(&model_info, request.response_time_ms)
|
||||
{
|
||||
return Ok(cost);
|
||||
}
|
||||
if request.provider == Some("perplexity") {
|
||||
return Ok(perplexity_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
if request.provider == Some("xai") {
|
||||
return Ok(xai_cost_per_token(request.usage, &model_info, request.at));
|
||||
}
|
||||
if matches!(
|
||||
request.provider,
|
||||
Some("dashscope" | "qwencloud" | "qwen_ai_platform")
|
||||
) {
|
||||
return Ok(dashscope_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
if request.provider == Some("fireworks_ai") {
|
||||
return Ok(fireworks_cost_per_token(
|
||||
request.usage,
|
||||
&self.entries[key],
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
if request.provider == Some("vertex_ai") {
|
||||
return Ok(vertex_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.service_tier,
|
||||
request.vertex_location,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
if !matches!(
|
||||
request.provider,
|
||||
match provider {
|
||||
Some(LlmProviders::PERPLEXITY) => {
|
||||
return Ok(perplexity_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
Some(LlmProviders::XAI) => {
|
||||
return Ok(xai_cost_per_token(request.usage, &model_info, request.at));
|
||||
}
|
||||
Some(
|
||||
"vertex_ai"
|
||||
| "anthropic"
|
||||
| "bedrock"
|
||||
| "openai"
|
||||
| "databricks"
|
||||
| "fireworks_ai"
|
||||
| "azure"
|
||||
| "gemini"
|
||||
| "deepseek"
|
||||
| "tencent"
|
||||
| "perplexity"
|
||||
| "xai"
|
||||
| "lemonade"
|
||||
| "dashscope"
|
||||
| "qwencloud"
|
||||
| "qwen_ai_platform"
|
||||
| "azure_ai"
|
||||
LlmProviders::DASHSCOPE | LlmProviders::QWENCLOUD | LlmProviders::QWEN_AI_PLATFORM,
|
||||
) => {
|
||||
return Ok(dashscope_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
Some(LlmProviders::FIREWORKS_AI) => {
|
||||
return Ok(fireworks_cost_per_token(
|
||||
request.usage,
|
||||
&self.entries[key],
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
Some(LlmProviders::VERTEX_AI) => {
|
||||
return Ok(vertex_cost_per_token(
|
||||
request.usage,
|
||||
&model_info,
|
||||
request.service_tier,
|
||||
request.vertex_location,
|
||||
request.at,
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
let dispatches_before_the_gate = matches!(
|
||||
provider,
|
||||
Some(
|
||||
LlmProviders::VERTEX_AI
|
||||
| LlmProviders::ANTHROPIC
|
||||
| LlmProviders::BEDROCK
|
||||
| LlmProviders::OPENAI
|
||||
| LlmProviders::DATABRICKS
|
||||
| LlmProviders::FIREWORKS_AI
|
||||
| LlmProviders::AZURE
|
||||
| LlmProviders::GEMINI
|
||||
| LlmProviders::DEEPSEEK
|
||||
| LlmProviders::TENCENT
|
||||
| LlmProviders::PERPLEXITY
|
||||
| LlmProviders::XAI
|
||||
| LlmProviders::LEMONADE
|
||||
| LlmProviders::DASHSCOPE
|
||||
| LlmProviders::QWENCLOUD
|
||||
| LlmProviders::QWEN_AI_PLATFORM
|
||||
| LlmProviders::AZURE_AI
|
||||
)
|
||||
) && !has_token_or_tiered_pricing(&model_info)
|
||||
{
|
||||
);
|
||||
if !dispatches_before_the_gate && !has_token_or_tiered_pricing(&model_info) {
|
||||
return Ok((0.0, 0.0));
|
||||
}
|
||||
let cost = calculate_generic_cost_from_model_info_with_region(
|
||||
|
|
@ -969,7 +978,7 @@ impl ModelInfoCatalog {
|
|||
request.vertex_location,
|
||||
request.at,
|
||||
);
|
||||
let speed = if request.provider == Some("anthropic") {
|
||||
let speed = if provider == Some(LlmProviders::ANTHROPIC) {
|
||||
fast_speed_multiplier(&model_info, request.usage)
|
||||
* get_provider_specific_geo_multiplier(
|
||||
&model_info,
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ pub mod batch;
|
|||
pub mod bedrock_image_cost;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod billed_token_rates;
|
||||
pub mod call_type;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod catalog;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
|
|
@ -78,6 +79,8 @@ pub mod pricing;
|
|||
#[allow(clippy::disallowed_types)]
|
||||
pub mod prompt_caching_savings;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod provider;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod provider_cache;
|
||||
#[allow(clippy::disallowed_types)]
|
||||
pub mod ptu_pricing;
|
||||
|
|
|
|||
514
litellm-rust/crates/cost/src/provider.rs
Normal file
514
litellm-rust/crates/cost/src/provider.rs
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
//! Mirrors Python's LlmProviders (litellm/types/utils.py), generated from its variant list.
|
||||
|
||||
// variant names mirror Python's enum members verbatim
|
||||
#![allow(non_camel_case_types)]
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub enum LlmProviders {
|
||||
OPENAI,
|
||||
CHATGPT,
|
||||
OPENAI_LIKE,
|
||||
JINA_AI,
|
||||
XAI,
|
||||
ZAI,
|
||||
CUSTOM_OPENAI,
|
||||
TEXT_COMPLETION_OPENAI,
|
||||
COHERE,
|
||||
COHERE_CHAT,
|
||||
CLARIFAI,
|
||||
ANTHROPIC,
|
||||
ANTHROPIC_TEXT,
|
||||
BYTEZ,
|
||||
REPLICATE,
|
||||
REDUCTO,
|
||||
AWS_TEXTRACT,
|
||||
RUNWAYML,
|
||||
AWS_POLLY,
|
||||
TRANSCRIBE,
|
||||
HUGGINGFACE,
|
||||
TOGETHER_AI,
|
||||
OPENROUTER,
|
||||
DATAROBOT,
|
||||
VERTEX_AI,
|
||||
VERTEX_AI_BETA,
|
||||
GEMINI,
|
||||
AI21,
|
||||
BASETEN,
|
||||
BLACK_FOREST_LABS,
|
||||
AZURE,
|
||||
AZURE_TEXT,
|
||||
AZURE_AI,
|
||||
SAGEMAKER,
|
||||
SAGEMAKER_CHAT,
|
||||
SAGEMAKER_NOVA,
|
||||
BEDROCK,
|
||||
VLLM,
|
||||
NLP_CLOUD,
|
||||
PETALS,
|
||||
OOBABOOGA,
|
||||
OLLAMA,
|
||||
OLLAMA_CHAT,
|
||||
DEEPINFRA,
|
||||
PERPLEXITY,
|
||||
MISTRAL,
|
||||
MILVUS,
|
||||
GROQ,
|
||||
A2A,
|
||||
GIGACHAT,
|
||||
NVIDIA_NIM,
|
||||
NVIDIA_RIVA,
|
||||
SONIOX,
|
||||
CEREBRAS,
|
||||
AI21_CHAT,
|
||||
VOLCENGINE,
|
||||
CODESTRAL,
|
||||
TEXT_COMPLETION_CODESTRAL,
|
||||
DASHSCOPE,
|
||||
QWENCLOUD,
|
||||
QWEN_AI_PLATFORM,
|
||||
MODELSCOPE,
|
||||
MOONSHOT,
|
||||
PUBLICAI,
|
||||
V0,
|
||||
MORPH,
|
||||
LAMBDA_AI,
|
||||
INCEPTION,
|
||||
TEXT_COMPLETION_INCEPTION,
|
||||
DEEPSEEK,
|
||||
SAMBANOVA,
|
||||
MARITALK,
|
||||
VOYAGE,
|
||||
CLOUDFLARE,
|
||||
XINFERENCE,
|
||||
FIREWORKS_AI,
|
||||
FRIENDLIAI,
|
||||
FEATHERLESS_AI,
|
||||
WATSONX,
|
||||
WATSONX_TEXT,
|
||||
TRITON,
|
||||
PREDIBASE,
|
||||
DATABRICKS,
|
||||
EMPOWER,
|
||||
GITHUB,
|
||||
RAGFLOW,
|
||||
COMPACTIFAI,
|
||||
DOCKER_MODEL_RUNNER,
|
||||
CUSTOM,
|
||||
LITELLM_PROXY,
|
||||
HOSTED_VLLM,
|
||||
TENCENT,
|
||||
LLAMAFILE,
|
||||
LM_STUDIO,
|
||||
GALADRIEL,
|
||||
NEBIUS,
|
||||
INFINITY,
|
||||
DEEPGRAM,
|
||||
ELEVENLABS,
|
||||
NOVITA,
|
||||
AIOHTTP_OPENAI,
|
||||
LANGFUSE,
|
||||
HUMANLOOP,
|
||||
TOPAZ,
|
||||
SAP_GENERATIVE_AI_HUB,
|
||||
ASSEMBLYAI,
|
||||
AZURE_SPEECH,
|
||||
CHARITY_ENGINE,
|
||||
GITHUB_COPILOT,
|
||||
SNOWFLAKE,
|
||||
GRADIENT_AI,
|
||||
LLAMA,
|
||||
NSCALE,
|
||||
PG_VECTOR,
|
||||
S3_VECTORS,
|
||||
VALKEY,
|
||||
MONGODB,
|
||||
HELICONE,
|
||||
HYPERBOLIC,
|
||||
RECRAFT,
|
||||
FAL_AI,
|
||||
STABILITY,
|
||||
HEROKU,
|
||||
AIML,
|
||||
COMETAPI,
|
||||
OCI,
|
||||
AUTO_ROUTER,
|
||||
VERCEL_AI_GATEWAY,
|
||||
EDENAI,
|
||||
DOTPROMPT,
|
||||
MANUS,
|
||||
WANDB,
|
||||
OVHCLOUD,
|
||||
SCALEWAY,
|
||||
LEMONADE,
|
||||
AMAZON_NOVA,
|
||||
A2A_AGENT,
|
||||
LANGGRAPH,
|
||||
LANGFLOW,
|
||||
MINIMAX,
|
||||
SYNTHETIC,
|
||||
APERTIS,
|
||||
NANOGPT,
|
||||
POE,
|
||||
CHUTES,
|
||||
NEOSANTARA,
|
||||
PARASAIL,
|
||||
XIAOMI_MIMO,
|
||||
TENSORMESH,
|
||||
LIBERTAI,
|
||||
PINSTRIPES,
|
||||
COGNITION,
|
||||
SCX_AI,
|
||||
DARKBLOOM,
|
||||
META,
|
||||
LITELLM_AGENT,
|
||||
CURSOR,
|
||||
BEDROCK_MANTLE,
|
||||
GDC,
|
||||
}
|
||||
|
||||
impl LlmProviders {
|
||||
pub const VALUES: [Self; 159] = [
|
||||
Self::OPENAI,
|
||||
Self::CHATGPT,
|
||||
Self::OPENAI_LIKE,
|
||||
Self::JINA_AI,
|
||||
Self::XAI,
|
||||
Self::ZAI,
|
||||
Self::CUSTOM_OPENAI,
|
||||
Self::TEXT_COMPLETION_OPENAI,
|
||||
Self::COHERE,
|
||||
Self::COHERE_CHAT,
|
||||
Self::CLARIFAI,
|
||||
Self::ANTHROPIC,
|
||||
Self::ANTHROPIC_TEXT,
|
||||
Self::BYTEZ,
|
||||
Self::REPLICATE,
|
||||
Self::REDUCTO,
|
||||
Self::AWS_TEXTRACT,
|
||||
Self::RUNWAYML,
|
||||
Self::AWS_POLLY,
|
||||
Self::TRANSCRIBE,
|
||||
Self::HUGGINGFACE,
|
||||
Self::TOGETHER_AI,
|
||||
Self::OPENROUTER,
|
||||
Self::DATAROBOT,
|
||||
Self::VERTEX_AI,
|
||||
Self::VERTEX_AI_BETA,
|
||||
Self::GEMINI,
|
||||
Self::AI21,
|
||||
Self::BASETEN,
|
||||
Self::BLACK_FOREST_LABS,
|
||||
Self::AZURE,
|
||||
Self::AZURE_TEXT,
|
||||
Self::AZURE_AI,
|
||||
Self::SAGEMAKER,
|
||||
Self::SAGEMAKER_CHAT,
|
||||
Self::SAGEMAKER_NOVA,
|
||||
Self::BEDROCK,
|
||||
Self::VLLM,
|
||||
Self::NLP_CLOUD,
|
||||
Self::PETALS,
|
||||
Self::OOBABOOGA,
|
||||
Self::OLLAMA,
|
||||
Self::OLLAMA_CHAT,
|
||||
Self::DEEPINFRA,
|
||||
Self::PERPLEXITY,
|
||||
Self::MISTRAL,
|
||||
Self::MILVUS,
|
||||
Self::GROQ,
|
||||
Self::A2A,
|
||||
Self::GIGACHAT,
|
||||
Self::NVIDIA_NIM,
|
||||
Self::NVIDIA_RIVA,
|
||||
Self::SONIOX,
|
||||
Self::CEREBRAS,
|
||||
Self::AI21_CHAT,
|
||||
Self::VOLCENGINE,
|
||||
Self::CODESTRAL,
|
||||
Self::TEXT_COMPLETION_CODESTRAL,
|
||||
Self::DASHSCOPE,
|
||||
Self::QWENCLOUD,
|
||||
Self::QWEN_AI_PLATFORM,
|
||||
Self::MODELSCOPE,
|
||||
Self::MOONSHOT,
|
||||
Self::PUBLICAI,
|
||||
Self::V0,
|
||||
Self::MORPH,
|
||||
Self::LAMBDA_AI,
|
||||
Self::INCEPTION,
|
||||
Self::TEXT_COMPLETION_INCEPTION,
|
||||
Self::DEEPSEEK,
|
||||
Self::SAMBANOVA,
|
||||
Self::MARITALK,
|
||||
Self::VOYAGE,
|
||||
Self::CLOUDFLARE,
|
||||
Self::XINFERENCE,
|
||||
Self::FIREWORKS_AI,
|
||||
Self::FRIENDLIAI,
|
||||
Self::FEATHERLESS_AI,
|
||||
Self::WATSONX,
|
||||
Self::WATSONX_TEXT,
|
||||
Self::TRITON,
|
||||
Self::PREDIBASE,
|
||||
Self::DATABRICKS,
|
||||
Self::EMPOWER,
|
||||
Self::GITHUB,
|
||||
Self::RAGFLOW,
|
||||
Self::COMPACTIFAI,
|
||||
Self::DOCKER_MODEL_RUNNER,
|
||||
Self::CUSTOM,
|
||||
Self::LITELLM_PROXY,
|
||||
Self::HOSTED_VLLM,
|
||||
Self::TENCENT,
|
||||
Self::LLAMAFILE,
|
||||
Self::LM_STUDIO,
|
||||
Self::GALADRIEL,
|
||||
Self::NEBIUS,
|
||||
Self::INFINITY,
|
||||
Self::DEEPGRAM,
|
||||
Self::ELEVENLABS,
|
||||
Self::NOVITA,
|
||||
Self::AIOHTTP_OPENAI,
|
||||
Self::LANGFUSE,
|
||||
Self::HUMANLOOP,
|
||||
Self::TOPAZ,
|
||||
Self::SAP_GENERATIVE_AI_HUB,
|
||||
Self::ASSEMBLYAI,
|
||||
Self::AZURE_SPEECH,
|
||||
Self::CHARITY_ENGINE,
|
||||
Self::GITHUB_COPILOT,
|
||||
Self::SNOWFLAKE,
|
||||
Self::GRADIENT_AI,
|
||||
Self::LLAMA,
|
||||
Self::NSCALE,
|
||||
Self::PG_VECTOR,
|
||||
Self::S3_VECTORS,
|
||||
Self::VALKEY,
|
||||
Self::MONGODB,
|
||||
Self::HELICONE,
|
||||
Self::HYPERBOLIC,
|
||||
Self::RECRAFT,
|
||||
Self::FAL_AI,
|
||||
Self::STABILITY,
|
||||
Self::HEROKU,
|
||||
Self::AIML,
|
||||
Self::COMETAPI,
|
||||
Self::OCI,
|
||||
Self::AUTO_ROUTER,
|
||||
Self::VERCEL_AI_GATEWAY,
|
||||
Self::EDENAI,
|
||||
Self::DOTPROMPT,
|
||||
Self::MANUS,
|
||||
Self::WANDB,
|
||||
Self::OVHCLOUD,
|
||||
Self::SCALEWAY,
|
||||
Self::LEMONADE,
|
||||
Self::AMAZON_NOVA,
|
||||
Self::A2A_AGENT,
|
||||
Self::LANGGRAPH,
|
||||
Self::LANGFLOW,
|
||||
Self::MINIMAX,
|
||||
Self::SYNTHETIC,
|
||||
Self::APERTIS,
|
||||
Self::NANOGPT,
|
||||
Self::POE,
|
||||
Self::CHUTES,
|
||||
Self::NEOSANTARA,
|
||||
Self::PARASAIL,
|
||||
Self::XIAOMI_MIMO,
|
||||
Self::TENSORMESH,
|
||||
Self::LIBERTAI,
|
||||
Self::PINSTRIPES,
|
||||
Self::COGNITION,
|
||||
Self::SCX_AI,
|
||||
Self::DARKBLOOM,
|
||||
Self::META,
|
||||
Self::LITELLM_AGENT,
|
||||
Self::CURSOR,
|
||||
Self::BEDROCK_MANTLE,
|
||||
Self::GDC,
|
||||
];
|
||||
|
||||
const NAMES: [(Self, &'static str); 159] = [
|
||||
(Self::OPENAI, "openai"),
|
||||
(Self::CHATGPT, "chatgpt"),
|
||||
(Self::OPENAI_LIKE, "openai_like"),
|
||||
(Self::JINA_AI, "jina_ai"),
|
||||
(Self::XAI, "xai"),
|
||||
(Self::ZAI, "zai"),
|
||||
(Self::CUSTOM_OPENAI, "custom_openai"),
|
||||
(Self::TEXT_COMPLETION_OPENAI, "text-completion-openai"),
|
||||
(Self::COHERE, "cohere"),
|
||||
(Self::COHERE_CHAT, "cohere_chat"),
|
||||
(Self::CLARIFAI, "clarifai"),
|
||||
(Self::ANTHROPIC, "anthropic"),
|
||||
(Self::ANTHROPIC_TEXT, "anthropic_text"),
|
||||
(Self::BYTEZ, "bytez"),
|
||||
(Self::REPLICATE, "replicate"),
|
||||
(Self::REDUCTO, "reducto"),
|
||||
(Self::AWS_TEXTRACT, "aws_textract"),
|
||||
(Self::RUNWAYML, "runwayml"),
|
||||
(Self::AWS_POLLY, "aws_polly"),
|
||||
(Self::TRANSCRIBE, "transcribe"),
|
||||
(Self::HUGGINGFACE, "huggingface"),
|
||||
(Self::TOGETHER_AI, "together_ai"),
|
||||
(Self::OPENROUTER, "openrouter"),
|
||||
(Self::DATAROBOT, "datarobot"),
|
||||
(Self::VERTEX_AI, "vertex_ai"),
|
||||
(Self::VERTEX_AI_BETA, "vertex_ai_beta"),
|
||||
(Self::GEMINI, "gemini"),
|
||||
(Self::AI21, "ai21"),
|
||||
(Self::BASETEN, "baseten"),
|
||||
(Self::BLACK_FOREST_LABS, "black_forest_labs"),
|
||||
(Self::AZURE, "azure"),
|
||||
(Self::AZURE_TEXT, "azure_text"),
|
||||
(Self::AZURE_AI, "azure_ai"),
|
||||
(Self::SAGEMAKER, "sagemaker"),
|
||||
(Self::SAGEMAKER_CHAT, "sagemaker_chat"),
|
||||
(Self::SAGEMAKER_NOVA, "sagemaker_nova"),
|
||||
(Self::BEDROCK, "bedrock"),
|
||||
(Self::VLLM, "vllm"),
|
||||
(Self::NLP_CLOUD, "nlp_cloud"),
|
||||
(Self::PETALS, "petals"),
|
||||
(Self::OOBABOOGA, "oobabooga"),
|
||||
(Self::OLLAMA, "ollama"),
|
||||
(Self::OLLAMA_CHAT, "ollama_chat"),
|
||||
(Self::DEEPINFRA, "deepinfra"),
|
||||
(Self::PERPLEXITY, "perplexity"),
|
||||
(Self::MISTRAL, "mistral"),
|
||||
(Self::MILVUS, "milvus"),
|
||||
(Self::GROQ, "groq"),
|
||||
(Self::A2A, "a2a"),
|
||||
(Self::GIGACHAT, "gigachat"),
|
||||
(Self::NVIDIA_NIM, "nvidia_nim"),
|
||||
(Self::NVIDIA_RIVA, "nvidia_riva"),
|
||||
(Self::SONIOX, "soniox"),
|
||||
(Self::CEREBRAS, "cerebras"),
|
||||
(Self::AI21_CHAT, "ai21_chat"),
|
||||
(Self::VOLCENGINE, "volcengine"),
|
||||
(Self::CODESTRAL, "codestral"),
|
||||
(Self::TEXT_COMPLETION_CODESTRAL, "text-completion-codestral"),
|
||||
(Self::DASHSCOPE, "dashscope"),
|
||||
(Self::QWENCLOUD, "qwencloud"),
|
||||
(Self::QWEN_AI_PLATFORM, "qwen_ai_platform"),
|
||||
(Self::MODELSCOPE, "modelscope"),
|
||||
(Self::MOONSHOT, "moonshot"),
|
||||
(Self::PUBLICAI, "publicai"),
|
||||
(Self::V0, "v0"),
|
||||
(Self::MORPH, "morph"),
|
||||
(Self::LAMBDA_AI, "lambda_ai"),
|
||||
(Self::INCEPTION, "inception"),
|
||||
(Self::TEXT_COMPLETION_INCEPTION, "text-completion-inception"),
|
||||
(Self::DEEPSEEK, "deepseek"),
|
||||
(Self::SAMBANOVA, "sambanova"),
|
||||
(Self::MARITALK, "maritalk"),
|
||||
(Self::VOYAGE, "voyage"),
|
||||
(Self::CLOUDFLARE, "cloudflare"),
|
||||
(Self::XINFERENCE, "xinference"),
|
||||
(Self::FIREWORKS_AI, "fireworks_ai"),
|
||||
(Self::FRIENDLIAI, "friendliai"),
|
||||
(Self::FEATHERLESS_AI, "featherless_ai"),
|
||||
(Self::WATSONX, "watsonx"),
|
||||
(Self::WATSONX_TEXT, "watsonx_text"),
|
||||
(Self::TRITON, "triton"),
|
||||
(Self::PREDIBASE, "predibase"),
|
||||
(Self::DATABRICKS, "databricks"),
|
||||
(Self::EMPOWER, "empower"),
|
||||
(Self::GITHUB, "github"),
|
||||
(Self::RAGFLOW, "ragflow"),
|
||||
(Self::COMPACTIFAI, "compactifai"),
|
||||
(Self::DOCKER_MODEL_RUNNER, "docker_model_runner"),
|
||||
(Self::CUSTOM, "custom"),
|
||||
(Self::LITELLM_PROXY, "litellm_proxy"),
|
||||
(Self::HOSTED_VLLM, "hosted_vllm"),
|
||||
(Self::TENCENT, "tencent"),
|
||||
(Self::LLAMAFILE, "llamafile"),
|
||||
(Self::LM_STUDIO, "lm_studio"),
|
||||
(Self::GALADRIEL, "galadriel"),
|
||||
(Self::NEBIUS, "nebius"),
|
||||
(Self::INFINITY, "infinity"),
|
||||
(Self::DEEPGRAM, "deepgram"),
|
||||
(Self::ELEVENLABS, "elevenlabs"),
|
||||
(Self::NOVITA, "novita"),
|
||||
(Self::AIOHTTP_OPENAI, "aiohttp_openai"),
|
||||
(Self::LANGFUSE, "langfuse"),
|
||||
(Self::HUMANLOOP, "humanloop"),
|
||||
(Self::TOPAZ, "topaz"),
|
||||
(Self::SAP_GENERATIVE_AI_HUB, "sap"),
|
||||
(Self::ASSEMBLYAI, "assemblyai"),
|
||||
(Self::AZURE_SPEECH, "azure_speech"),
|
||||
(Self::CHARITY_ENGINE, "charity_engine"),
|
||||
(Self::GITHUB_COPILOT, "github_copilot"),
|
||||
(Self::SNOWFLAKE, "snowflake"),
|
||||
(Self::GRADIENT_AI, "gradient_ai"),
|
||||
(Self::LLAMA, "meta_llama"),
|
||||
(Self::NSCALE, "nscale"),
|
||||
(Self::PG_VECTOR, "pg_vector"),
|
||||
(Self::S3_VECTORS, "s3_vectors"),
|
||||
(Self::VALKEY, "valkey"),
|
||||
(Self::MONGODB, "mongodb"),
|
||||
(Self::HELICONE, "helicone"),
|
||||
(Self::HYPERBOLIC, "hyperbolic"),
|
||||
(Self::RECRAFT, "recraft"),
|
||||
(Self::FAL_AI, "fal_ai"),
|
||||
(Self::STABILITY, "stability"),
|
||||
(Self::HEROKU, "heroku"),
|
||||
(Self::AIML, "aiml"),
|
||||
(Self::COMETAPI, "cometapi"),
|
||||
(Self::OCI, "oci"),
|
||||
(Self::AUTO_ROUTER, "auto_router"),
|
||||
(Self::VERCEL_AI_GATEWAY, "vercel_ai_gateway"),
|
||||
(Self::EDENAI, "edenai"),
|
||||
(Self::DOTPROMPT, "dotprompt"),
|
||||
(Self::MANUS, "manus"),
|
||||
(Self::WANDB, "wandb"),
|
||||
(Self::OVHCLOUD, "ovhcloud"),
|
||||
(Self::SCALEWAY, "scaleway"),
|
||||
(Self::LEMONADE, "lemonade"),
|
||||
(Self::AMAZON_NOVA, "amazon_nova"),
|
||||
(Self::A2A_AGENT, "a2a_agent"),
|
||||
(Self::LANGGRAPH, "langgraph"),
|
||||
(Self::LANGFLOW, "langflow"),
|
||||
(Self::MINIMAX, "minimax"),
|
||||
(Self::SYNTHETIC, "synthetic"),
|
||||
(Self::APERTIS, "apertis"),
|
||||
(Self::NANOGPT, "nano-gpt"),
|
||||
(Self::POE, "poe"),
|
||||
(Self::CHUTES, "chutes"),
|
||||
(Self::NEOSANTARA, "neosantara"),
|
||||
(Self::PARASAIL, "parasail"),
|
||||
(Self::XIAOMI_MIMO, "xiaomi_mimo"),
|
||||
(Self::TENSORMESH, "tensormesh"),
|
||||
(Self::LIBERTAI, "libertai"),
|
||||
(Self::PINSTRIPES, "pinstripes"),
|
||||
(Self::COGNITION, "cognition"),
|
||||
(Self::SCX_AI, "scx-ai"),
|
||||
(Self::DARKBLOOM, "darkbloom"),
|
||||
(Self::META, "meta"),
|
||||
(Self::LITELLM_AGENT, "litellm_agent"),
|
||||
(Self::CURSOR, "cursor"),
|
||||
(Self::BEDROCK_MANTLE, "bedrock_mantle"),
|
||||
(Self::GDC, "gdc"),
|
||||
];
|
||||
|
||||
pub fn as_str(self) -> &'static str {
|
||||
Self::NAMES
|
||||
.iter()
|
||||
.find(|(variant, _)| *variant == self)
|
||||
.map(|(_, name)| *name)
|
||||
.expect("every variant carries its name")
|
||||
}
|
||||
|
||||
pub fn parse(name: &str) -> Option<Self> {
|
||||
Self::NAMES
|
||||
.iter()
|
||||
.find(|(_, known)| *known == name)
|
||||
.map(|(variant, _)| *variant)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for LlmProviders {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
|
@ -1116,11 +1116,15 @@ def pinning_surface() -> dict[str, Any]:
|
|||
"result": result,
|
||||
}
|
||||
)
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
|
||||
return {
|
||||
"thresholds": thresholds,
|
||||
"service_tier_suffixes": list(cost_utils._SERVICE_TIER_SUFFIXES),
|
||||
"batch_tier_key": cost_utils._BATCH_TIER_KEY.pattern,
|
||||
"cost_per_unit": cost_per_unit,
|
||||
"llm_providers": [provider.value for provider in LlmProviders],
|
||||
"call_types": [call.value for call in CallTypes],
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"python_commit": "a8725892a4d1986e09e6db7a736f872083479de1",
|
||||
"python_commit": "0f4ce30103244d670c2ba275dfaa599cca99fbf2",
|
||||
"generic": [
|
||||
{
|
||||
"id": "audio_mixed_modalities",
|
||||
|
|
@ -2322,6 +2322,318 @@
|
|||
"default": null,
|
||||
"result": null
|
||||
}
|
||||
],
|
||||
"llm_providers": [
|
||||
"openai",
|
||||
"chatgpt",
|
||||
"openai_like",
|
||||
"jina_ai",
|
||||
"xai",
|
||||
"zai",
|
||||
"custom_openai",
|
||||
"text-completion-openai",
|
||||
"cohere",
|
||||
"cohere_chat",
|
||||
"clarifai",
|
||||
"anthropic",
|
||||
"anthropic_text",
|
||||
"bytez",
|
||||
"replicate",
|
||||
"reducto",
|
||||
"aws_textract",
|
||||
"runwayml",
|
||||
"aws_polly",
|
||||
"transcribe",
|
||||
"huggingface",
|
||||
"together_ai",
|
||||
"openrouter",
|
||||
"datarobot",
|
||||
"vertex_ai",
|
||||
"vertex_ai_beta",
|
||||
"gemini",
|
||||
"ai21",
|
||||
"baseten",
|
||||
"black_forest_labs",
|
||||
"azure",
|
||||
"azure_text",
|
||||
"azure_ai",
|
||||
"sagemaker",
|
||||
"sagemaker_chat",
|
||||
"sagemaker_nova",
|
||||
"bedrock",
|
||||
"vllm",
|
||||
"nlp_cloud",
|
||||
"petals",
|
||||
"oobabooga",
|
||||
"ollama",
|
||||
"ollama_chat",
|
||||
"deepinfra",
|
||||
"perplexity",
|
||||
"mistral",
|
||||
"milvus",
|
||||
"groq",
|
||||
"a2a",
|
||||
"gigachat",
|
||||
"nvidia_nim",
|
||||
"nvidia_riva",
|
||||
"soniox",
|
||||
"cerebras",
|
||||
"ai21_chat",
|
||||
"volcengine",
|
||||
"codestral",
|
||||
"text-completion-codestral",
|
||||
"dashscope",
|
||||
"qwencloud",
|
||||
"qwen_ai_platform",
|
||||
"modelscope",
|
||||
"moonshot",
|
||||
"publicai",
|
||||
"v0",
|
||||
"morph",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"text-completion-inception",
|
||||
"deepseek",
|
||||
"sambanova",
|
||||
"maritalk",
|
||||
"voyage",
|
||||
"cloudflare",
|
||||
"xinference",
|
||||
"fireworks_ai",
|
||||
"friendliai",
|
||||
"featherless_ai",
|
||||
"watsonx",
|
||||
"watsonx_text",
|
||||
"triton",
|
||||
"predibase",
|
||||
"databricks",
|
||||
"empower",
|
||||
"github",
|
||||
"ragflow",
|
||||
"compactifai",
|
||||
"docker_model_runner",
|
||||
"custom",
|
||||
"litellm_proxy",
|
||||
"hosted_vllm",
|
||||
"tencent",
|
||||
"llamafile",
|
||||
"lm_studio",
|
||||
"galadriel",
|
||||
"nebius",
|
||||
"infinity",
|
||||
"deepgram",
|
||||
"elevenlabs",
|
||||
"novita",
|
||||
"aiohttp_openai",
|
||||
"langfuse",
|
||||
"humanloop",
|
||||
"topaz",
|
||||
"sap",
|
||||
"assemblyai",
|
||||
"azure_speech",
|
||||
"charity_engine",
|
||||
"github_copilot",
|
||||
"snowflake",
|
||||
"gradient_ai",
|
||||
"meta_llama",
|
||||
"nscale",
|
||||
"pg_vector",
|
||||
"s3_vectors",
|
||||
"valkey",
|
||||
"mongodb",
|
||||
"helicone",
|
||||
"hyperbolic",
|
||||
"recraft",
|
||||
"fal_ai",
|
||||
"stability",
|
||||
"heroku",
|
||||
"aiml",
|
||||
"cometapi",
|
||||
"oci",
|
||||
"auto_router",
|
||||
"vercel_ai_gateway",
|
||||
"edenai",
|
||||
"dotprompt",
|
||||
"manus",
|
||||
"wandb",
|
||||
"ovhcloud",
|
||||
"scaleway",
|
||||
"lemonade",
|
||||
"amazon_nova",
|
||||
"a2a_agent",
|
||||
"langgraph",
|
||||
"langflow",
|
||||
"minimax",
|
||||
"synthetic",
|
||||
"apertis",
|
||||
"nano-gpt",
|
||||
"poe",
|
||||
"chutes",
|
||||
"neosantara",
|
||||
"parasail",
|
||||
"xiaomi_mimo",
|
||||
"tensormesh",
|
||||
"libertai",
|
||||
"pinstripes",
|
||||
"cognition",
|
||||
"scx-ai",
|
||||
"darkbloom",
|
||||
"meta",
|
||||
"litellm_agent",
|
||||
"cursor",
|
||||
"bedrock_mantle",
|
||||
"gdc"
|
||||
],
|
||||
"call_types": [
|
||||
"embedding",
|
||||
"aembedding",
|
||||
"completion",
|
||||
"acompletion",
|
||||
"atext_completion",
|
||||
"text_completion",
|
||||
"image_generation",
|
||||
"aimage_generation",
|
||||
"image_edit",
|
||||
"aimage_edit",
|
||||
"moderation",
|
||||
"amoderation",
|
||||
"atranscription",
|
||||
"transcription",
|
||||
"aspeech",
|
||||
"speech",
|
||||
"rerank",
|
||||
"arerank",
|
||||
"search",
|
||||
"asearch",
|
||||
"_arealtime",
|
||||
"_aresponses_websocket",
|
||||
"create_batch",
|
||||
"acreate_batch",
|
||||
"aretrieve_batch",
|
||||
"retrieve_batch",
|
||||
"acancel_batch",
|
||||
"cancel_batch",
|
||||
"pass_through_endpoint",
|
||||
"anthropic_messages",
|
||||
"aanthropic_messages",
|
||||
"get_assistants",
|
||||
"aget_assistants",
|
||||
"create_assistants",
|
||||
"acreate_assistants",
|
||||
"delete_assistant",
|
||||
"adelete_assistant",
|
||||
"acreate_thread",
|
||||
"create_thread",
|
||||
"aget_thread",
|
||||
"get_thread",
|
||||
"a_add_message",
|
||||
"add_message",
|
||||
"aget_messages",
|
||||
"get_messages",
|
||||
"arun_thread",
|
||||
"run_thread",
|
||||
"arun_thread_stream",
|
||||
"run_thread_stream",
|
||||
"afile_retrieve",
|
||||
"file_retrieve",
|
||||
"afile_delete",
|
||||
"file_delete",
|
||||
"afile_list",
|
||||
"file_list",
|
||||
"acreate_file",
|
||||
"create_file",
|
||||
"afile_content",
|
||||
"file_content",
|
||||
"create_fine_tuning_job",
|
||||
"acreate_fine_tuning_job",
|
||||
"create_video",
|
||||
"acreate_video",
|
||||
"video_generation",
|
||||
"avideo_generation",
|
||||
"avideo_retrieve",
|
||||
"video_retrieve",
|
||||
"avideo_content",
|
||||
"video_content",
|
||||
"video_remix",
|
||||
"avideo_remix",
|
||||
"video_list",
|
||||
"avideo_list",
|
||||
"video_retrieve_job",
|
||||
"avideo_retrieve_job",
|
||||
"video_delete",
|
||||
"avideo_delete",
|
||||
"video_create_character",
|
||||
"avideo_create_character",
|
||||
"video_get_character",
|
||||
"avideo_get_character",
|
||||
"video_edit",
|
||||
"avideo_edit",
|
||||
"video_extension",
|
||||
"avideo_extension",
|
||||
"vector_store_file_create",
|
||||
"avector_store_file_create",
|
||||
"vector_store_file_list",
|
||||
"avector_store_file_list",
|
||||
"vector_store_file_retrieve",
|
||||
"avector_store_file_retrieve",
|
||||
"vector_store_file_content",
|
||||
"avector_store_file_content",
|
||||
"vector_store_file_update",
|
||||
"avector_store_file_update",
|
||||
"vector_store_file_delete",
|
||||
"avector_store_file_delete",
|
||||
"vector_store_create",
|
||||
"avector_store_create",
|
||||
"vector_store_search",
|
||||
"avector_store_search",
|
||||
"ingest",
|
||||
"aingest",
|
||||
"query",
|
||||
"aquery",
|
||||
"create_interaction",
|
||||
"acreate_interaction",
|
||||
"create_container",
|
||||
"acreate_container",
|
||||
"list_containers",
|
||||
"alist_containers",
|
||||
"retrieve_container",
|
||||
"aretrieve_container",
|
||||
"delete_container",
|
||||
"adelete_container",
|
||||
"list_container_files",
|
||||
"alist_container_files",
|
||||
"upload_container_file",
|
||||
"aupload_container_file",
|
||||
"create_sandbox",
|
||||
"acreate_sandbox",
|
||||
"delete_sandbox",
|
||||
"adelete_sandbox",
|
||||
"run_code",
|
||||
"arun_code",
|
||||
"code_interpreter_tool",
|
||||
"acode_interpreter_tool",
|
||||
"acancel_fine_tuning_job",
|
||||
"cancel_fine_tuning_job",
|
||||
"alist_fine_tuning_jobs",
|
||||
"list_fine_tuning_jobs",
|
||||
"aretrieve_fine_tuning_job",
|
||||
"retrieve_fine_tuning_job",
|
||||
"responses",
|
||||
"aresponses",
|
||||
"alist_input_items",
|
||||
"llm_passthrough_route",
|
||||
"allm_passthrough_route",
|
||||
"generate_content",
|
||||
"agenerate_content",
|
||||
"generate_content_stream",
|
||||
"agenerate_content_stream",
|
||||
"ocr",
|
||||
"aocr",
|
||||
"call_mcp_tool",
|
||||
"list_mcp_tools",
|
||||
"asend_message",
|
||||
"send_message",
|
||||
"acreate_skill"
|
||||
]
|
||||
},
|
||||
"model_info_projection": {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,9 @@
|
|||
|
||||
use serde_json::Value;
|
||||
|
||||
use litellm_cost::call_type::CallTypes;
|
||||
use litellm_cost::pricing::{Metric, ServiceTier, tokenize};
|
||||
use litellm_cost::provider::LlmProviders;
|
||||
use litellm_cost::wire::{RawCatalogEntry, cost_per_unit};
|
||||
|
||||
fn fixture() -> Value {
|
||||
|
|
@ -172,3 +174,45 @@ fn registered_entry_parses_into_model_pricing_with_projected_rates() {
|
|||
Some(6e-7)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_and_call_type_enums_mirror_python() {
|
||||
let pinning = pinning();
|
||||
let python_providers: std::collections::BTreeSet<&str> = pinning["llm_providers"]
|
||||
.as_array()
|
||||
.expect("llm providers")
|
||||
.iter()
|
||||
.map(|value| value.as_str().expect("provider value"))
|
||||
.collect();
|
||||
let rust_providers: std::collections::BTreeSet<&str> = LlmProviders::VALUES
|
||||
.iter()
|
||||
.map(|provider| provider.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
rust_providers, python_providers,
|
||||
"every Python LlmProviders value is a Rust variant and no extras exist"
|
||||
);
|
||||
for provider in &python_providers {
|
||||
assert!(LlmProviders::parse(provider).is_some());
|
||||
}
|
||||
let python_calls: std::collections::BTreeSet<&str> = pinning["call_types"]
|
||||
.as_array()
|
||||
.expect("call types")
|
||||
.iter()
|
||||
.map(|value| value.as_str().expect("call type value"))
|
||||
.collect();
|
||||
let rust_calls: std::collections::BTreeSet<&str> =
|
||||
CallTypes::VALUES.iter().map(|call| call.as_str()).collect();
|
||||
assert_eq!(
|
||||
rust_calls, python_calls,
|
||||
"every Python CallTypes value is a Rust variant and no extras exist"
|
||||
);
|
||||
for call in &python_calls {
|
||||
assert!(CallTypes::parse(call).is_some());
|
||||
}
|
||||
assert_eq!(CallTypes::parse("_arealtime"), Some(CallTypes::arealtime));
|
||||
assert_eq!(
|
||||
CallTypes::parse("_aresponses_websocket"),
|
||||
Some(CallTypes::aresponses_websocket)
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue