scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge remote-tracking branch 'origin/main' into beluga_decrease_anys_20261107

Browse source 
# Conflicts:
#	litellm/llms/anthropic/messages/adapters/transformation.py
This commit is contained in:
mateo-berri 2026-11-07 23:25:32 +00:00
commit c226a70b1d
93 changed files with 6257 additions and 3046 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm-rust/crates/inference-messages/src/types.rs
View file

@ -144,6 +144,7 @@ mod tests {
supports_reasoning: true,
supports_adaptive_thinking: true,
thinking_always_on: false,
thinking_on_by_default: false,
supports_legacy_thinking: false,
supports_output_config: true,
supports_sampling_params: false,

1
litellm-rust/crates/llms/src/anthropic/common_utils.rs
View file

@ -1913,6 +1913,7 @@ mod tests {
supports_reasoning: false,
supports_adaptive_thinking: false,
thinking_always_on: false,
thinking_on_by_default: false,
supports_legacy_thinking: false,
supports_output_config: false,
supports_sampling_params: true,

37
litellm-rust/crates/llms/src/anthropic/messages/thinking.rs
View file

@ -2,8 +2,8 @@ use litellm_llms_types::{
formats::{
chat_completions::ReasoningEffort,
messages::{
AdaptiveThinking, EffortLevel, MessagesOptionalParams, MessagesRequest, OutputConfig,
ThinkingConfig, ThinkingDisplay,
AdaptiveThinking, DisabledThinking, EffortLevel, MessagesOptionalParams,
MessagesRequest, OutputConfig, ThinkingConfig, ThinkingDisplay,
},
},
recognized::Recognized,
@ -172,7 +172,11 @@ fn translate_reasoning_effort(
) else {
return Ok(MessagesRequest {
params: MessagesOptionalParams {
thinking: None,
thinking: (context.capabilities.thinking_on_by_default
&& !context.capabilities.thinking_always_on)
.then(|| {
Recognized::Known(ThinkingConfig::Disabled(DisabledThinking::default()))
}),
output_config: None,
..request.params
},
@ -748,6 +752,33 @@ mod tests {
);
}
#[rstest]
#[case::default_thinking(true, false, Some(serde_json::json!({"type": "disabled"}))) ]
#[case::legacy(false, false, None)]
#[case::always_on(true, true, None)]
fn none_disables_only_optional_default_thinking(
#[case] thinking_on_by_default: bool,
#[case] thinking_always_on: bool,
#[case] thinking: Option<Value>,
) {
let capabilities = MessagesModelCapabilities {
thinking_on_by_default,
thinking_always_on,
..opus_4_7()
};
let expected = match thinking {
Some(thinking) => serde_json::json!({"max_tokens": 1024, "thinking": thinking}),
None => serde_json::json!({"max_tokens": 1024}),
};
assert_eq!(
translate(
capabilities,
serde_json::json!({"max_tokens": 1024, "reasoning_effort": "none"})
),
Ok(request(expected))
);
}
#[rstest]
#[case::adaptive_model(opus_4_7())]
#[case::effort_capable_model(opus_4_5())]

3
litellm-rust/crates/llms/src/base_llm/messages/context.rs
View file

@ -32,6 +32,8 @@ pub struct MessagesModelCapabilities {
#[serde(default)]
pub thinking_always_on: bool,
#[serde(default)]
pub thinking_on_by_default: bool,
#[serde(default)]
pub supports_legacy_thinking: bool,
#[serde(default)]
pub supports_output_config: bool,
@ -55,6 +57,7 @@ impl Default for MessagesModelCapabilities {
supports_reasoning: false,
supports_adaptive_thinking: false,
thinking_always_on: false,
thinking_on_by_default: false,
supports_legacy_thinking: false,
supports_output_config: false,
supports_sampling_params: true,

87
litellm-rust/crates/model-catalog/src/catalog.rs
View file

@ -142,6 +142,7 @@ impl Catalog {
let mut sections = Vec::new();
let mut entries = IndexMap::with_capacity(document.len());
let mut alias_lists = Vec::new();
let mut pointer_rows = Vec::new();
for (index, (name, value)) in document.into_iter().enumerate() {
let fields = match value {
Value::Object(fields) if !is_section(&name) => fields,
@ -157,16 +158,34 @@ impl Catalog {
continue;
}
};
let (model, aliases, row_problems) = read_row(&fields);
let row = read_row(&fields);
let readable = row.problems.is_empty();
problems.extend(
row_problems
row.problems
.into_iter()
.map(|problem| problem.prepend(&name)),
);
if let Some(aliases) = aliases {
if let Some(target) = row.alias_of {
if readable {
pointer_rows.push((name.clone(), target));
}
sections.push(Section {
index,
key: name,
value: Value::Object(fields),
});
continue;
}
if let Some(aliases) = row.aliases {
alias_lists.push((name.clone(), aliases));
}
entries.insert(name, ModelEntry { fields, model });
entries.insert(
name,
ModelEntry {
fields,
model: row.model,
},
);
}
if entries.is_empty() {
return Err(Error::new("", Repr::Empty));
@ -190,6 +209,21 @@ impl Catalog {
}
}
}
for (alias, target) in pointer_rows {
if !entries.contains_key(&target) {
problems.push(Error::from_key(&alias, Repr::AliasTargetMissing { target }));
} else if let Some(taken_by) = aliases.get(&alias) {
problems.push(Error::from_key(
&alias,
Repr::AliasTaken {
alias: alias.clone(),
taken_by: taken_by.clone(),
},
));
} else {
aliases.insert(alias, target);
}
}
let fallback_generalizations = sections
.iter()
@ -242,8 +276,8 @@ impl Catalog {
}
pub fn row_errors(model: &str, fields: &Fields) -> Vec<Error> {
let (_, _, errors) = read_row(fields);
errors
read_row(fields)
.problems
.into_iter()
.map(|error| error.prepend(model))
.collect()
@ -255,7 +289,13 @@ impl Catalog {
(0..self.rows().entries.len() + self.sections.len()).map_while(move |index| match sections
.next_if(|section| section.index == index)
{
Some(section) => Some((section.key.as_str(), Written::Section(&section.value))),
Some(section) => Some((
section.key.as_str(),
match &section.value {
Value::Object(fields) if !is_section(&section.key) => Written::Entry(fields),
value => Written::Section(value),
},
)),
None => entries
.next()
.map(|(name, entry)| (name.as_str(), Written::Entry(&entry.fields))),
@ -406,16 +446,41 @@ impl Catalog {
}
}
fn read_row(fields: &Fields) -> (Option<Model>, Option<Vec<String>>, Vec<Error>) {
struct RowRead {
model: Option<Model>,
aliases: Option<Vec<String>>,
alias_of: Option<String>,
problems: Vec<Error>,
}
fn read_row(fields: &Fields) -> RowRead {
let wire = EntryWire::read(fields);
if let Some(target) = wire.attributes.alias_of.clone() {
let other_fields = fields
.keys()
.filter(|key| *key != "alias_of")
.map(|key| Error::from_key(key, Repr::AliasRowField));
return RowRead {
model: None,
aliases: None,
alias_of: Some(target),
problems: wire.problems.into_iter().chain(other_fields).collect(),
};
}
let aliases = wire.attributes.aliases.clone();
let timezone_error = unknown_timezone(&wire);
match Model::try_from(wire) {
Ok(model) => (Some(model), aliases, timezone_error.into_iter().collect()),
let (model, problems) = match Model::try_from(wire) {
Ok(model) => (Some(model), timezone_error.into_iter().collect()),
Err(mut errors) => {
errors.extend(timezone_error);
(None, aliases, errors)
(None, errors)
}
};
RowRead {
model,
aliases,
alias_of: None,
problems,
}
}

10
litellm-rust/crates/model-catalog/src/error.rs
View file

@ -67,6 +67,12 @@ pub(crate) enum Repr {
#[error("alias {alias:?} already belongs to {taken_by}")]
#[strum_discriminants(strum(serialize = "alias_taken"))]
AliasTaken { alias: String, taken_by: String },
#[error("alias_of {target:?} is not a model name")]
#[strum_discriminants(strum(serialize = "alias_target_missing"))]
AliasTargetMissing { target: String },
#[error("an alias_of row carries no other field")]
#[strum_discriminants(strum(serialize = "alias_row_field"))]
AliasRowField,
#[error("{first} and {second} price the same meter under the same conditions")]
#[strum_discriminants(strum(serialize = "duplicate_price"))]
DuplicatePrice { first: String, second: String },
@ -141,6 +147,8 @@ impl Error {
| Repr::UnknownTimezone { .. }
| Repr::AliasShadowsModel { .. }
| Repr::AliasTaken { .. }
| Repr::AliasTargetMissing { .. }
| Repr::AliasRowField
| Repr::DuplicatePrice { .. }
| Repr::SuffixInsideBlock { .. }
| Repr::TierRange
@ -327,6 +335,8 @@ mod tests {
#[case::unknown_timezone(Repr::UnknownTimezone { zone: "invalid".into() }, Rejects::Row)]
#[case::alias_shadows_model(Repr::AliasShadowsModel { alias: "model".into() }, Rejects::Row)]
#[case::alias_taken(Repr::AliasTaken { alias: "alias".into(), taken_by: "owner".into() }, Rejects::Row)]
#[case::alias_target_missing(Repr::AliasTargetMissing { target: "owner".into() }, Rejects::Row)]
#[case::alias_row_field(Repr::AliasRowField, Rejects::Row)]
#[case::duplicate_price(Repr::DuplicatePrice { first: "a".into(), second: "b".into() }, Rejects::Row)]
#[case::suffix_inside_block(Repr::SuffixInsideBlock { block: "tiered_pricing" }, Rejects::Row)]
#[case::tier_range(Repr::TierRange, Rejects::Row)]

12
litellm-rust/crates/model-catalog/src/schema.rs
View file

@ -104,9 +104,18 @@ pub fn registry_json_schema() -> Schema {
"title".into(),
"LiteLLM model prices and context window registry".into(),
);
definitions.insert(
"aliasRow".into(),
json!({
"type": "object",
"required": ["alias_of"],
"properties": {"alias_of": {"type": "string"}},
"additionalProperties": false,
}),
);
root.insert(
"additionalProperties".into(),
json!({"$ref": "#/$defs/modelEntry"}),
json!({"anyOf": [{"$ref": "#/$defs/modelEntry"}, {"$ref": "#/$defs/aliasRow"}]}),
);
root.insert("$defs".into(), Value::Object(definitions));
Schema::from(root.clone())
@ -153,6 +162,7 @@ fn decorate_model_entry(schema: &mut Value) {
let rates = json!({FLAT_KEY_PATTERN: {"type": "number", "minimum": 0}});
let object = schema.as_object_mut().unwrap();
object.insert("required".into(), json!(["litellm_provider"]));
object.insert("not".into(), json!({"required": ["alias_of"]}));
object.insert("additionalProperties".into(), Value::Bool(true));
object.insert(
"patternProperties".into(),

2
litellm-rust/crates/model-catalog/src/wire/attributes.rs
View file

@ -47,6 +47,8 @@ macro_rules! attributes {
}
attributes! {
/// Model this row is another name for; such a row carries no other key.
alias_of: String,
/// Other names this entry is served under.
aliases: Vec<String>,
audio_transcription_config: String,

79
litellm-rust/crates/model-catalog/tests/catalog.rs
View file

@ -139,6 +139,18 @@ fn a_bad_entry_is_reported_by_model_and_key_and_kept_raw() {
json!({"First": {"litellm_provider": "t", "aliases": ["shared"]}, "Second": {"litellm_provider": "t", "aliases": ["shared"]}}),
r#"Second: alias "shared" already belongs to First"#
)]
#[case::alias_of_target_missing(
json!({"Real": {"litellm_provider": "t"}, "ghost-latest": {"alias_of": "Ghost"}}),
r#"ghost-latest: alias_of "Ghost" is not a model name"#
)]
#[case::alias_of_row_with_other_fields(
json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real", "mode": "chat"}}),
"real-latest.mode: an alias_of row carries no other field"
)]
#[case::alias_of_claimed_by_a_list(
json!({"Real": {"litellm_provider": "t"}, "real-latest": {"alias_of": "Real"}, "Other": {"litellm_provider": "t", "aliases": ["real-latest"]}}),
r#"real-latest: alias "real-latest" already belongs to Other"#
)]
#[case::aliases_not_a_list(
json!({"a": {"litellm_provider": "t", "aliases": "bad"}}),
"a.aliases: invalid value: invalid type: string \"bad\", expected a sequence"
@ -399,3 +411,70 @@ fn a_bad_value_keeps_its_json_error_as_the_source_and_names_its_key_and_field()
assert_eq!(error.key().as_deref(), Some("open/router"));
assert_eq!(error.field().as_deref(), Some("input_cost_per_token"));
}
#[rstest]
fn an_alias_of_row_is_served_as_its_target_and_written_as_itself() {
let catalog = catalog(json!({
"Real": {"litellm_provider": "t", "input_cost_per_token": 1e-6},
"real-latest": {"alias_of": "Real"},
"After": {"litellm_provider": "t"}
}));
let found = catalog.lookup("REAL-LATEST").unwrap();
assert_eq!(
(found.matched_key, found.canonical_key),
("real-latest", "Real")
);
assert_eq!(
found.entry.field("input_cost_per_token"),
Some(&json!(1e-6))
);
assert_eq!(catalog.model_count(), 2);
assert_eq!(
catalog.served().map(|(key, _)| key).collect::<Vec<_>>(),
["Real", "After", "real-latest"]
);
assert_eq!(
written(&catalog),
r#"{"Real":{"litellm_provider":"t","input_cost_per_token":1e-6},"real-latest":{"alias_of":"Real"},"After":{"litellm_provider":"t"}}"#
);
}
#[rstest]
fn a_dangling_alias_of_row_is_rejected_and_dropped_from_the_written_document() {
let parsed = loaded(json!({
"Real": {"litellm_provider": "t"},
"ghost-latest": {"alias_of": "Ghost"}
}));
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(removed.len(), 1);
assert!(catalog.lookup("ghost-latest").is_none());
assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#);
}
#[rstest]
fn an_alias_of_row_with_other_fields_is_never_served() {
let parsed = loaded(json!({
"Real": {"litellm_provider": "t"},
"real-latest": {"alias_of": "Real", "mode": "chat"}
}));
assert!(parsed.value.lookup("real-latest").is_none());
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(removed.len(), 1);
assert!(catalog.lookup("real-latest").is_none());
assert_eq!(written(&catalog), r#"{"Real":{"litellm_provider":"t"}}"#);
}
#[rstest]
fn a_pointer_row_that_repeats_a_listed_alias_is_dropped_and_the_model_is_kept() {
let parsed = loaded(json!({
"Real": {"litellm_provider": "t", "aliases": ["real-latest"]},
"real-latest": {"alias_of": "Real"}
}));
let (catalog, removed) = parsed.value.without_rejected(&parsed.errors);
assert_eq!(removed.len(), 1);
assert_eq!(catalog.lookup("real-latest").unwrap().canonical_key, "Real");
assert_eq!(
written(&catalog),
r#"{"Real":{"litellm_provider":"t","aliases":["real-latest"]}}"#
);
}

4
litellm-rust/crates/model-catalog/tests/schema.rs
View file

@ -35,6 +35,7 @@ fn generated_registry_schema_validates_checked_in_catalog() {
#[case(json!({"example": {"litellm_provider": "test"}}))]
#[case(json!({"example": {"litellm_provider": "test", "future_field": true}}))]
#[case(json!({"sample_spec": {"litellm_provider": "placeholder"}}))]
#[case(json!({"example": {"litellm_provider": "test"}, "example-latest": {"alias_of": "example"}}))]
fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value) {
assert!(registry_validator().is_valid(&document));
}
@ -54,6 +55,9 @@ fn generated_registry_schema_keeps_reader_compatibility(#[case] document: Value)
#[case::empty_windows(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": []}}))]
#[case::invalid_weekday(json!({"litellm_provider": "test", "off_peak_pricing": {"windows": [{"hours_utc": "00:00-01:00", "weekdays": [0]}]}}))]
#[case::invalid_aliases(json!({"litellm_provider": "test", "aliases": "wrong"}))]
#[case::alias_of_on_a_model(json!({"litellm_provider": "test", "alias_of": "other"}))]
#[case::alias_row_with_other_fields(json!({"alias_of": "other", "mode": "chat"}))]
#[case::alias_of_not_a_string(json!({"alias_of": 4}))]
#[case::non_object_model(json!(4))]
fn generated_registry_schema_rejects_invalid_entries(#[case] entry: Value) {
assert!(!registry_validator().is_valid(&json!({"example": entry})));

253
litellm/__init__.py
View file

@ -81,7 +81,6 @@ from litellm.constants import (
import httpx
from openai import DefaultAsyncHttpxClient, DefaultHttpxClient
# register_async_client_cleanup is lazy-loaded and called on first access
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
@ -98,7 +97,6 @@ failure_callback: List[CALLBACK_TYPES] = []
service_callback: List[CALLBACK_TYPES] = []
audit_log_callbacks: List[CALLBACK_TYPES] = []
batch_line_item_callbacks: Sequence[CALLBACK_TYPES] = ()
# logging_callback_manager is lazy-loaded via __getattr__
_custom_logger_compatible_callbacks_literal = Literal[
"lago",
"openmeter",
@ -507,8 +505,6 @@ skill_search_embedding_model: Optional[str] = None
public_model_groups_links: dict[str, str | dict[str, object]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
# priority_reservation_settings is lazy-loaded via __getattr__
# Only declare for type checking - at runtime __getattr__ handles it
if TYPE_CHECKING:
priority_reservation_settings: Optional["PriorityReservationSettings"] = None
@ -543,8 +539,6 @@ _google_kms_resource_name: Optional[str] = None
_key_management_system: Optional["KeyManagementSystem"] = None
# Note: KeyManagementSettings must be eagerly imported because _key_management_settings
# is accessed during import time in secret_managers/main.py
# We'll import it after the lazy import system is set up
# We can't define it here because KeyManagementSettings is lazy-loaded
#### PII MASKING ####
output_parse_pii: bool = False
#############################################
@ -1196,8 +1190,6 @@ model_list = list(
model_list_set = set(model_list)
# provider_list is lazy-loaded via __getattr__ to avoid importing LlmProviders at import time
def _build_models_by_provider() -> dict:
return {
@ -1378,10 +1370,6 @@ all_embedding_models = (
####### VIDEO GENERATION MODELS ###################
openai_video_generation_models = ["sora-2"]
# timeout is lazy-loaded via __getattr__
# get_llm_provider is lazy-loaded via __getattr__
# remove_index_from_tool_calls is lazy-loaded via __getattr__
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
from litellm.types.secret_managers.main import KeyManagementSettings
@ -1391,8 +1379,6 @@ _key_management_settings: KeyManagementSettings = KeyManagementSettings()
# client must be imported immediately as it's used as a decorator at function definition time
from .utils import client
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
# (which imports tiktoken) at import time
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
@ -1416,12 +1402,9 @@ from .llms.bedrock.embed.amazon_titan_v2_transformation import (
)
from .llms.topaz.common_utils import TopazModelInfo
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
@ -1650,8 +1633,6 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
global_gitlab_config = config
# Lazy loading system for heavy modules to reduce initial import time and memory usage
if TYPE_CHECKING:
from . import guardrails as guardrails
from litellm.types.utils import ModelInfo as _ModelInfoType
@ -1659,7 +1640,6 @@ if TYPE_CHECKING:
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.caching.caching import Cache
# Type stubs for lazy-loaded configs to help mypy
from .llms.bedrock.chat.converse_transformation import (
AmazonConverseConfig as AmazonConverseConfig,
)
@ -2088,7 +2068,6 @@ if TYPE_CHECKING:
)
from .llms.gdc.chat.transformation import GDCGeminiConfig as GDCGeminiConfig
# Type stubs for lazy-loaded config instances
openaiOSeriesConfig: OpenAIOSeriesConfig
openAIGPTConfig: OpenAIGPTConfig
openAIGPTAudioConfig: OpenAIGPTAudioConfig
@ -2146,7 +2125,6 @@ if TYPE_CHECKING:
VertexGeminiConfig as _VertexGeminiConfig,
)
# Type stubs for lazy-loaded config classes (to help mypy understand types)
VLLMConfig: Type[_VLLMConfig]
DeepSeekChatConfig: Type[_DeepSeekChatConfig]
TencentChatConfig: Type[_TencentChatConfig]
@ -2423,63 +2401,34 @@ if TYPE_CHECKING:
module_level_aclient: AsyncHTTPHandler
module_level_client: HTTPHandler
# Bedrock tool name mappings instance (lazy-loaded)
from litellm.caching.caching import InMemoryCache
bedrock_tool_name_mappings: InMemoryCache
# Azure exception class (lazy-loaded)
from litellm.llms.azure.common_utils import AzureOpenAIError
# Secret manager types (lazy-loaded)
from litellm.types.secret_managers.main import (
KeyManagementSystem,
KeyManagementSettings, # Not lazy-loaded - needed for _key_management_settings initialization
)
# Custom logger class (lazy-loaded)
from litellm.integrations.custom_logger import CustomLogger
# Datadog LLM observability params (lazy-loaded)
from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams
# Logging callback manager class and instance (lazy-loaded)
from litellm.litellm_core_utils.logging_callback_manager import (
LoggingCallbackManager,
)
logging_callback_manager: LoggingCallbackManager
# provider_list is lazy-loaded
from litellm.types.utils import LlmProviders
provider_list: List[Union[LlmProviders, str]]
# Note: AmazonConverseConfig and OpenAILikeChatConfig are imported above in TYPE_CHECKING block
# Track if async client cleanup has been registered (for lazy loading)
_async_client_cleanup_registered = False
_AGENT_EXPORTS: Final = frozenset(
{
"agent",
"aagent",
"agent_session",
"aagent_session",
"agent_resume",
"aagent_resume",
"agent_capabilities",
"Harness",
"ClaudeCodeOptions",
"CodexOptions",
"OpenCodeOptions",
"PiOptions",
"DeepAgentsOptions",
"ToolLoopOptions",
}
)
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
@ -2492,9 +2441,7 @@ if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes",
def __getattr__(name: str) -> Any:
"""Lazy import handler with cached registry for improved performance."""
global _async_client_cleanup_registered
# Register async client cleanup on first access (only once)
if not _async_client_cleanup_registered:
from litellm.llms.custom_httpx.async_client_cleanup import (
register_async_client_cleanup,
@ -2503,205 +2450,9 @@ def __getattr__(name: str) -> Any:
register_async_client_cleanup()
_async_client_cleanup_registered = True
# Use cached registry from _lazy_imports instead of importing tuples every time
from ._lazy_imports import get_lazy_import_registry
from ._lazy_imports import resolve_lazy_attribute
registry: Final = get_lazy_import_registry()
return resolve_lazy_attribute(__name__, name)
# Check if name is in registry and call the cached handler function
if name in registry:
handler_func: Final = registry[name]
return handler_func(name)
if name in ("proxy", "guardrails"):
import importlib
return importlib.import_module(f"litellm.{name}")
if name == "harness" or name in _AGENT_EXPORTS:
import importlib
harness_module = importlib.import_module("litellm.harness")
return harness_module if name == "harness" else getattr(harness_module, name)
# Lazy load encoding from main.py to avoid heavy tiktoken import
if name == "encoding":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "encoding" not in _globals:
from .main import encoding as _encoding
_globals["encoding"] = _encoding
return _globals["encoding"]
# Lazy load bedrock_tool_name_mappings instance
if name == "bedrock_tool_name_mappings":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "bedrock_tool_name_mappings" not in _globals:
from .llms.bedrock.chat.invoke_handler import (
bedrock_tool_name_mappings as _bedrock_tool_name_mappings,
)
_globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings
return _globals["bedrock_tool_name_mappings"]
# Lazy load AzureOpenAIError exception class
if name == "AzureOpenAIError":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "AzureOpenAIError" not in _globals:
from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
_globals["AzureOpenAIError"] = _AzureOpenAIError
return _globals["AzureOpenAIError"]
# Lazy load openaiOSeriesConfig instance
if name == "openaiOSeriesConfig":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
if "openaiOSeriesConfig" not in _globals:
# Import the config class and instantiate it
config_class = __getattr__("OpenAIOSeriesConfig")
_globals["openaiOSeriesConfig"] = config_class()
return _globals["openaiOSeriesConfig"]
# Lazy load other config instances
_config_instances: Final = {
"openAIGPTConfig": "OpenAIGPTConfig",
"openAIGPTAudioConfig": "OpenAIGPTAudioConfig",
"openAIGPT5Config": "OpenAIGPT5Config",
"nvidiaNimConfig": "NvidiaNimConfig",
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
}
if name in _config_instances:
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
if name not in _globals:
# Import the config class and instantiate it
config_class = __getattr__(_config_instances[name])
_globals[name] = config_class()
return _globals[name]
# Handle OpenAIO1Config alias
if name == "OpenAIO1Config":
return __getattr__("OpenAIOSeriesConfig")
# Lazy load provider_list
if name == "provider_list":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "provider_list" not in _globals:
# LlmProviders is eagerly imported above, so we can import it directly
from litellm.types.utils import LlmProviders
_globals["provider_list"] = list(LlmProviders)
return _globals["provider_list"]
# Lazy load priority_reservation_settings instance
if name == "priority_reservation_settings":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "priority_reservation_settings" not in _globals:
# Import the class and instantiate it
PriorityReservationSettings: Final = __getattr__("PriorityReservationSettings")
_globals["priority_reservation_settings"] = PriorityReservationSettings()
return _globals["priority_reservation_settings"]
# Lazy load logging_callback_manager instance
if name == "logging_callback_manager":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "logging_callback_manager" not in _globals:
# Import the class and instantiate it
LoggingCallbackManager: Final = __getattr__("LoggingCallbackManager")
_globals["logging_callback_manager"] = LoggingCallbackManager()
return _globals["logging_callback_manager"]
# Lazy load _service_logger module
if name == "_service_logger":
from ._lazy_imports import get_litellm_globals
_globals = get_litellm_globals()
# Check if already cached
if "_service_logger" not in _globals:
# Import the module lazily
import litellm._service_logger
_globals["_service_logger"] = litellm._service_logger
return _globals["_service_logger"]
# Lazy load evals module functions
if name in [
"acreate_eval",
"alist_evals",
"aget_eval",
"aupdate_eval",
"adelete_eval",
"acancel_eval",
"create_eval",
"list_evals",
"get_eval",
"update_eval",
"delete_eval",
"cancel_eval",
"acreate_run",
"alist_runs",
"aget_run",
"acancel_run",
"adelete_run",
"create_run",
"list_runs",
"get_run",
"cancel_run",
"delete_run",
]:
from litellm.evals.main import (
acreate_eval,
alist_evals,
aget_eval,
aupdate_eval,
adelete_eval,
acancel_eval,
create_eval,
list_evals,
get_eval,
update_eval,
delete_eval,
cancel_eval,
acreate_run,
alist_runs,
aget_run,
acancel_run,
adelete_run,
create_run,
list_runs,
get_run,
cancel_run,
delete_run,
)
from types import MappingProxyType
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
return evals_exports[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
mark_litellm_import_complete()

19
litellm/_internal_context.py
View file

@ -44,6 +44,8 @@ _service_caller: Final[ContextVar[str | None]] = ContextVar("service_caller", de
_mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=False)
_resent_on: Final[ContextVar[Callable[[Exception], bool] | None]] = ContextVar("resent_on", default=None)
_request_parent_otel_span: Final[ContextVar[Span | None]] = ContextVar("request_parent_otel_span", default=None)
@ -168,6 +170,23 @@ def in_mirrored_call() -> bool:
return _mirrored_call.get()
@contextmanager
def resent_on(rejection: Callable[[Exception], bool]) -> Generator[None]:
"""An attempt its caller sends again when it fails with ``rejection``. That failure is not the request's
outcome, so it is neither logged as the request's failure nor retried, and its budget reservation and rate-limit
slot stay with the attempt sent again."""
token: Final = _resent_on.set(rejection)
try:
yield
finally:
_restore(_resent_on, token)
def is_resent_failure(error: Exception) -> bool:
rejection: Final = _resent_on.get()
return rejection is not None and rejection(error)
def set_request_parent_otel_span(span: Span) -> None:
"""Anchor the proxy request's OTEL span for datastore service spans that no call site passes a parent to.
Never reset: it dies with the request task and rides the tasks it spawns, like ``request.state``."""

503
litellm/_lazy_imports.py
View file

@ -1,60 +1,12 @@
"""
Lazy Import System
This module implements lazy loading for LiteLLM attributes. Instead of importing
everything when the module loads, we only import things when they're actually used.
How it works:
1. When someone accesses `litellm.some_attribute`, Python calls __getattr__ in __init__.py
2. __getattr__ looks up the attribute name in a registry
3. The registry points to a handler function (like _lazy_import_utils)
4. The handler function imports the module and returns the attribute
5. The result is cached so we don't import it again
This makes importing litellm much faster because we don't load heavy dependencies
until they're actually needed.
"""
"""Resolve the attributes of `litellm` and `litellm.utils` that are imported on first access."""
import importlib
import sys
from collections.abc import Callable, Mapping
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final
from types import MappingProxyType, ModuleType
from typing import TYPE_CHECKING, Final
from typing_extensions import ReadOnly, TypedDict
# Import all the data structures that define what can be lazy-loaded
# These are just lists of names and maps of where to find them
from ._lazy_imports_registry import (
# Import maps
_BEDROCK_TYPES_IMPORT_MAP,
_CACHING_IMPORT_MAP,
_COST_CALCULATOR_IMPORT_MAP,
_DOTPROMPT_IMPORT_MAP,
_LITELLM_LOGGING_IMPORT_MAP,
_LLM_CONFIGS_IMPORT_MAP,
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
_TOKEN_COUNTER_IMPORT_MAP,
_TYPES_IMPORT_MAP,
_TYPES_UTILS_IMPORT_MAP,
_UTILS_IMPORT_MAP,
_UTILS_MODULE_IMPORT_MAP,
# Name tuples
BEDROCK_TYPES_NAMES,
CACHING_NAMES,
COST_CALCULATOR_NAMES,
DOTPROMPT_NAMES,
HTTP_HANDLER_NAMES,
LITELLM_LOGGING_NAMES,
LLM_CLIENT_CACHE_NAMES,
LLM_CONFIG_NAMES,
LLM_PROVIDER_LOGIC_NAMES,
TOKEN_COUNTER_NAMES,
TYPES_NAMES,
TYPES_UTILS_NAMES,
UTILS_MODULE_NAMES,
UTILS_NAMES,
)
from ._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES
if TYPE_CHECKING:
import httpx
@ -62,35 +14,6 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
def get_litellm_globals() -> dict[str, object]:
"""
Get the globals dictionary of the litellm module.
This is where we cache imported attributes so we don't import them twice.
When you do `litellm.some_function`, it gets stored in this dictionary.
"""
return sys.modules["litellm"].__dict__
def _get_utils_globals() -> dict[str, object]:
"""
Get the globals dictionary of the utils module.
This is where we cache imported attributes so we don't import them twice.
When you do `litellm.utils.some_function`, it gets stored in this dictionary.
"""
return sys.modules["litellm.utils"].__dict__
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
"""Read the configured `litellm.request_timeout` used for the module level http clients."""
return litellm_globals.get("request_timeout")
# These are special lazy loaders for things that are used internally
# They're separate from the main lazy import system because they have specific use cases
def get_default_encoding() -> "Tokenizer":
from litellm.rust_bridge.tokenizer import get_encoding
@ -100,381 +23,125 @@ def get_default_encoding() -> "Tokenizer":
_get_default_encoding = get_default_encoding
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
_get_modified_max_tokens_func: "Callable[..., int | None] | None" = None
def _get_modified_max_tokens() -> "Callable[..., int | None]":
"""
Lazily load and cache the get_modified_max_tokens function.
from litellm.litellm_core_utils.token_counter import get_modified_max_tokens
This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time.
The function is cached after the first import.
This is used internally by utils.py functions that need the token counter but shouldn't
trigger its import during module load.
"""
global _get_modified_max_tokens_func
if _get_modified_max_tokens_func is None:
from litellm.litellm_core_utils.token_counter import (
get_modified_max_tokens as _get_modified_max_tokens_imported,
)
_get_modified_max_tokens_func = _get_modified_max_tokens_imported
return _get_modified_max_tokens_func
# Lazy loader for token_counter to avoid importing token_counter module at module import time
_token_counter_new_func: "Callable[..., int] | None" = None
_messages_reach_token_count_func: "Callable[..., bool] | None" = None
return get_modified_max_tokens
def get_token_counter_new() -> "Callable[..., int]":
"""
Lazily load and cache the token_counter function (aliased as token_counter_new).
from litellm.litellm_core_utils.token_counter import token_counter
This avoids importing `litellm.litellm_core_utils.token_counter` at `litellm` import time.
The function is cached after the first import.
This is used internally by utils.py functions that need the token counter but shouldn't
trigger its import during module load.
"""
global _token_counter_new_func
if _token_counter_new_func is None:
from litellm.litellm_core_utils.token_counter import (
token_counter as _token_counter_imported,
)
_token_counter_new_func = _token_counter_imported
return _token_counter_new_func
return token_counter
_get_token_counter_new = get_token_counter_new
def get_messages_reach_token_count() -> "Callable[..., bool]":
"""Lazily load ``messages_reach_token_count`` for the same reason as ``get_token_counter_new``."""
global _messages_reach_token_count_func
if _messages_reach_token_count_func is None:
from litellm.litellm_core_utils.token_counter import (
messages_reach_token_count as _messages_reach_token_count_imported,
)
from litellm.litellm_core_utils.token_counter import messages_reach_token_count
_messages_reach_token_count_func = _messages_reach_token_count_imported
return _messages_reach_token_count_func
return messages_reach_token_count
_get_messages_reach_token_count = get_messages_reach_token_count
# ============================================================================
# MAIN LAZY IMPORT SYSTEM
# ============================================================================
# This registry maps attribute names (like "ModelResponse") to handler functions
# It's built once the first time someone accesses a lazy-loaded attribute
# Example: {"ModelResponse": _lazy_import_utils, "Cache": _lazy_import_caching, ...}
_LAZY_IMPORT_REGISTRY: dict[str, Callable[[str], object]] | None = None
_NOT_LOADED: Final = object()
def get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
"""
Build the registry that maps attribute names to their handler functions.
This is called once, the first time someone accesses a lazy-loaded attribute.
After that, we just look up the handler function in this dictionary.
Returns:
Dictionary like {"ModelResponse": _lazy_import_utils, ...}
"""
global _LAZY_IMPORT_REGISTRY
if _LAZY_IMPORT_REGISTRY is None:
# Build the registry by going through each category and mapping
# all the names in that category to their handler function
_LAZY_IMPORT_REGISTRY = {}
# For each category, map all its names to the handler function
# Example: All names in UTILS_NAMES get mapped to _lazy_import_utils
for name in COST_CALCULATOR_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_cost_calculator
for name in LITELLM_LOGGING_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_litellm_logging
for name in UTILS_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils
for name in TOKEN_COUNTER_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_token_counter
for name in LLM_CLIENT_CACHE_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_client_cache
for name in BEDROCK_TYPES_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_bedrock_types
for name in TYPES_UTILS_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_types_utils
for name in CACHING_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_caching
for name in HTTP_HANDLER_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_http_handlers
for name in DOTPROMPT_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_dotprompt
for name in LLM_CONFIG_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_configs
for name in TYPES_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_types
for name in LLM_PROVIDER_LOGIC_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
for name in UTILS_MODULE_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
return _LAZY_IMPORT_REGISTRY
def _cached(namespace: str, name: str) -> object:
return vars(sys.modules[namespace]).get(name, _NOT_LOADED)
_get_lazy_import_registry = get_lazy_import_registry
class _AttributeView(TypedDict):
"""Holds one module attribute so the lazily fetched value is read back as ``object``."""
value: ReadOnly[object]
def _module_attribute(module: ModuleType, attr_name: str) -> object:
attribute: Final[_AttributeView] = {"value": getattr(module, attr_name)}
return attribute["value"]
def _generic_lazy_import(name: str, import_map: Mapping[str, tuple[str, str]], category: str) -> object:
"""
Generic function that handles lazy importing for most attributes.
This is the workhorse function - it does the actual importing and caching.
Most handler functions just call this with their specific import map.
Steps:
1. Check if the name exists in the import map (if not, raise error)
2. Check if we've already imported it (if yes, return cached value)
3. Look up where to find it (module_path and attr_name from the map)
4. Import the module (Python caches this automatically)
5. Get the attribute from the module
6. Cache it in _globals so we don't import again
7. Return it
Args:
name: The attribute name someone is trying to access (e.g., "ModelResponse")
import_map: Dictionary telling us where to find each attribute
Format: {"ModelResponse": (".utils", "ModelResponse")}
category: Just for error messages (e.g., "Utils", "Cost calculator")
"""
# Step 1: Make sure this attribute exists in our map
if name not in import_map:
raise AttributeError(f"{category} lazy import: unknown attribute {name!r}")
# Step 2: Get the cache (where we store imported things)
_globals: Final = get_litellm_globals()
# Step 3: If we've already imported it, just return the cached version
if name in _globals:
return _globals[name]
# Step 4: Look up where to find this attribute
# The map tells us: (module_path, attribute_name)
# Example: (".utils", "ModelResponse") means "look in .utils module, get ModelResponse"
module_path, attr_name = import_map[name]
# Step 5: Import the module
# Python automatically caches modules in sys.modules, so calling this twice is fast
# If module_path starts with ".", it's a relative import (needs package="litellm")
# Otherwise it's an absolute import (like "litellm.caching.caching")
if module_path.startswith("."):
module = importlib.import_module(module_path, package="litellm")
else:
module = importlib.import_module(module_path)
# Step 6: Get the actual attribute from the module
# Example: getattr(utils_module, "ModelResponse") returns the ModelResponse class
value: Final = _module_attribute(module, attr_name)
# Step 7: Cache it so we don't have to import again next time
_globals[name] = value
# Step 8: Return it
def _cache(namespace: str, name: str, value: object) -> object:
setattr(sys.modules[namespace], name, value)
return value
# ============================================================================
# HANDLER FUNCTIONS
# ============================================================================
# These functions are called when someone accesses a lazy-loaded attribute.
# Most of them just call _generic_lazy_import with their specific import map.
# The registry (above) maps attribute names to these handler functions.
def _import(namespace: str, name: str, location: tuple[str, str]) -> object:
cached: Final = _cached(namespace, name)
if cached is not _NOT_LOADED:
return cached
module_path, attr_name = location
module: Final[ModuleType] = importlib.import_module(module_path, package="litellm")
attribute: Final[object] = getattr(module, attr_name)
return _cache(namespace, name, attribute)
def _lazy_import_utils(name: str) -> object:
"""Handler for utils module attributes (ModelResponse, token_counter, etc.)"""
return _generic_lazy_import(name, _UTILS_IMPORT_MAP, "Utils")
def _instance_of(class_name: str) -> Callable[[], object]:
def create() -> object:
cls: Final = resolve_lazy_attribute("litellm", class_name)
assert isinstance(cls, type)
return cls()
return create
def _lazy_import_cost_calculator(name: str) -> object:
"""Handler for cost calculator functions (completion_cost, cost_per_token, etc.)"""
return _generic_lazy_import(name, _COST_CALCULATOR_IMPORT_MAP, "Cost calculator")
def _submodule(module_path: str) -> Callable[[], object]:
return lambda: importlib.import_module(module_path)
def _lazy_import_token_counter(name: str) -> object:
"""Handler for token counter utilities"""
return _generic_lazy_import(name, _TOKEN_COUNTER_IMPORT_MAP, "Token counter")
def _request_timeout() -> "float | httpx.Timeout | None":
import httpx
timeout: Final = _cached("litellm", "request_timeout")
return timeout if isinstance(timeout, (int, float, httpx.Timeout)) else None
def _lazy_import_bedrock_types(name: str) -> object:
"""Handler for Bedrock type aliases"""
return _generic_lazy_import(name, _BEDROCK_TYPES_IMPORT_MAP, "Bedrock types")
def _module_level_aclient() -> object:
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
return get_async_httpx_client(
llm_provider="litellm_module_level_client",
params={"timeout": _request_timeout(), "client_alias": "module level aclient"},
)
def _lazy_import_types_utils(name: str) -> object:
"""Handler for types from litellm.types.utils (BudgetConfig, ImageObject, etc.)"""
return _generic_lazy_import(name, _TYPES_UTILS_IMPORT_MAP, "Types utils")
def _module_level_client() -> object:
from litellm.llms.custom_httpx.http_handler import HTTPHandler
return HTTPHandler(timeout=_request_timeout())
def _lazy_import_caching(name: str) -> object:
"""Handler for caching classes (Cache, DualCache, RedisCache, etc.)"""
return _generic_lazy_import(name, _CACHING_IMPORT_MAP, "Caching")
def _provider_list() -> object:
from litellm.types.utils import LlmProviders
return list(LlmProviders)
def _lazy_import_dotprompt(name: str) -> object:
"""Handler for dotprompt integration globals"""
return _generic_lazy_import(name, _DOTPROMPT_IMPORT_MAP, "Dotprompt")
_LITELLM_FACTORIES: Final[Mapping[str, Callable[[], object]]] = MappingProxyType(
{
"in_memory_llm_clients_cache": _instance_of("LLMClientCache"),
"module_level_aclient": _module_level_aclient,
"module_level_client": _module_level_client,
"openaiOSeriesConfig": _instance_of("OpenAIOSeriesConfig"),
"openAIGPTConfig": _instance_of("OpenAIGPTConfig"),
"openAIGPTAudioConfig": _instance_of("OpenAIGPTAudioConfig"),
"openAIGPT5Config": _instance_of("OpenAIGPT5Config"),
"nvidiaNimConfig": _instance_of("NvidiaNimConfig"),
"nvidiaNimEmbeddingConfig": _instance_of("NvidiaNimEmbeddingConfig"),
"priority_reservation_settings": _instance_of("PriorityReservationSettings"),
"logging_callback_manager": _instance_of("LoggingCallbackManager"),
"provider_list": _provider_list,
"proxy": _submodule("litellm.proxy"),
"guardrails": _submodule("litellm.guardrails"),
"harness": _submodule("litellm.harness"),
"_service_logger": _submodule("litellm._service_logger"),
}
)
def _lazy_import_types(name: str) -> object:
"""Handler for type classes (GuardrailItem, etc.)"""
return _generic_lazy_import(name, _TYPES_IMPORT_MAP, "Types")
def _lazy_import_llm_configs(name: str) -> object:
"""Handler for LLM config classes (AnthropicConfig, OpenAILikeChatConfig, etc.)"""
return _generic_lazy_import(name, _LLM_CONFIGS_IMPORT_MAP, "LLM config")
def _lazy_import_litellm_logging(name: str) -> object:
"""Handler for litellm_logging module (Logging, modify_integration)"""
return _generic_lazy_import(name, _LITELLM_LOGGING_IMPORT_MAP, "Litellm logging")
def _lazy_import_llm_provider_logic(name: str) -> object:
"""Handler for LLM provider logic functions (get_llm_provider, etc.)"""
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
def _lazy_import_utils_module(name: str) -> object:
"""
Handler for utils module lazy imports.
This uses a custom implementation because utils module needs to use
_get_utils_globals() instead of get_litellm_globals() for caching.
"""
# Check if this attribute exists in our map
if name not in _UTILS_MODULE_IMPORT_MAP:
raise AttributeError(f"Utils module lazy import: unknown attribute {name!r}")
# Get the cache (where we store imported things) - use utils globals
_globals: Final = _get_utils_globals()
# If we've already imported it, just return the cached version
if name in _globals:
return _globals[name]
# Look up where to find this attribute
module_path, attr_name = _UTILS_MODULE_IMPORT_MAP[name]
# Import the module
if module_path.startswith("."):
module = importlib.import_module(module_path, package="litellm")
else:
module = importlib.import_module(module_path)
# Get the actual attribute from the module
value: Final = _module_attribute(module, attr_name)
# Cache it so we don't have to import again next time
_globals[name] = value
# Return it
return value
# ============================================================================
# SPECIAL HANDLERS
# ============================================================================
# These handlers have custom logic that doesn't fit the generic pattern
def _lazy_import_llm_client_cache(name: str) -> object:
"""
Handler for LLM client cache - has special logic for singleton instance.
This one is different because:
- "LLMClientCache" is the class itself
- "in_memory_llm_clients_cache" is a singleton instance of that class
So we need custom logic to handle both cases.
"""
_globals: Final = get_litellm_globals()
# If already cached, return it
if name in _globals:
return _globals[name]
# Import the class
from litellm.caching.llm_caching_handler import LLMClientCache
# If they want the class itself, return it
if name == "LLMClientCache":
_globals["LLMClientCache"] = LLMClientCache
return LLMClientCache
# If they want the singleton instance, create it (only once)
if name == "in_memory_llm_clients_cache":
instance: Final = LLMClientCache()
_globals["in_memory_llm_clients_cache"] = instance
return instance
raise AttributeError(f"LLM client cache lazy import: unknown attribute {name!r}")
def _lazy_import_http_handlers(name: str) -> object:
"""
Handler for HTTP clients - has special logic for creating client instances.
This one is different because:
- These aren't just imports, they're actual client instances that need to be created
- They need configuration (timeout, etc.) from the module globals
- They use factory functions instead of direct instantiation
"""
_globals: Final = get_litellm_globals()
if name == "module_level_aclient":
# Create an async HTTP client using the factory function
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
# Get timeout from module config (if set)
async_timeout: Final = _get_module_level_client_timeout(_globals)
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
# Create the client instance
provider_id: Final = "litellm_module_level_client"
async_client: Final = get_async_httpx_client(
llm_provider=provider_id,
params=params,
)
# Cache it so we don't create it again
_globals["module_level_aclient"] = async_client
return async_client
if name == "module_level_client":
# Create a sync HTTP client
from litellm.llms.custom_httpx.http_handler import HTTPHandler
sync_timeout: Final = _get_module_level_client_timeout(_globals)
sync_client: Final = HTTPHandler(timeout=sync_timeout)
# Cache it
_globals["module_level_client"] = sync_client
return sync_client
raise AttributeError(f"HTTP handlers lazy import: unknown attribute {name!r}")
def resolve_lazy_attribute(module_name: str, name: str) -> object:
"""Import, cache and return `name`, the way `module_name.__getattr__` would."""
litellm_location: Final = LITELLM_ATTRIBUTES.get(name)
if litellm_location is not None:
return _import("litellm", name, litellm_location)
utils_location: Final = UTILS_ATTRIBUTES.get(name)
if utils_location is not None:
return _import("litellm.utils", name, utils_location)
factory: Final = _LITELLM_FACTORIES.get(name) if module_name == "litellm" else None
if factory is None:
raise AttributeError(f"module {module_name!r} has no attribute {name!r}")
cached: Final = _cached("litellm", name)
return cached if cached is not _NOT_LOADED else _cache("litellm", name, factory())

2312
litellm/_lazy_imports_registry.py
View file

File diff suppressed because it is too large Load diff

8
litellm/completion_extras/litellm_responses_transformation/handler.py
View file

@ -182,6 +182,9 @@ class ResponsesToCompletionBridgeHandler:
model_response: Final = validated_kwargs["model_response"]
logging_obj: Final = validated_kwargs["logging_obj"]
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
from .transformation import serves_anthropic_messages
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
@ -262,6 +265,7 @@ class ResponsesToCompletionBridgeHandler:
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
replays_openai_history=custom_llm_provider == "openai",
streams_reasoning_signatures=streams_reasoning_signatures,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -284,6 +288,9 @@ class ResponsesToCompletionBridgeHandler:
model_response: Final = validated_kwargs["model_response"]
logging_obj: Final = validated_kwargs["logging_obj"]
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
from .transformation import serves_anthropic_messages
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
logging_obj.record_bridged_chat_request(optional_params, litellm_params)
@ -367,6 +374,7 @@ class ResponsesToCompletionBridgeHandler:
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
replays_openai_history=custom_llm_provider == "openai",
streams_reasoning_signatures=streams_reasoning_signatures,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,

108
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -39,7 +39,12 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_or_none,
drops_unsupported_params,
iter_items,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_with_readable_mode,
@ -48,6 +53,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
responses_reasoning_items_from_thinking_blocks,
thinking_blocks_from_reasoning_items,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
@ -69,6 +75,8 @@ from litellm.responses.utils import (
from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParamFunctionChunk,
@ -77,7 +85,7 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
@ -93,8 +101,6 @@ if TYPE_CHECKING:
AllMessageValues,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
ChatCompletionToolReferenceObject,
OpenAIMessageContentListBlock,
)
@ -264,6 +270,10 @@ class _BuiltReasoningItem(TypedDict):
summary: Sequence[_ReasoningSummaryText]
def serves_anthropic_messages(logging_obj: object) -> bool:
return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value
def _get_reasoning_items(
msg: "AllMessageValues",
) -> list[ChatCompletionReasoningItem]:
@ -496,6 +506,77 @@ def _as_chat_reasoning_items(
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
def _with_stream_block_index(
block: Mapping[str, object],
index: int,
summary_streamed: bool,
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
if block.get("type") == "redacted_thinking":
data: Final = block.get("data")
return ChatCompletionRedactedThinkingBlock(
type="redacted_thinking",
data=data if isinstance(data, str) else "",
index=index,
)
thinking: Final = block.get("thinking")
signature: Final = block.get("signature")
return ChatCompletionThinkingBlock(
type="thinking",
thinking=thinking if isinstance(thinking, str) and not summary_streamed else "",
signature=signature if isinstance(signature, str) else None,
index=index,
)
def _stream_output_index(chunk: Mapping[str, object]) -> int | None:
output_index: Final = chunk.get("output_index")
return output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else None
def _reasoning_done_thinking_blocks(
item: Mapping[str, object],
output_index: int | None,
summary_streamed: bool,
) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]:
index: Final = output_index if output_index is not None else 0
return tuple(
_with_stream_block_index(block, index, summary_streamed)
for block in thinking_blocks_from_reasoning_items((item,))
)
def _reasoning_done_stream_chunk(
chunk: Mapping[str, object],
summary_streamed_outputs: frozenset[int | None],
) -> ModelResponseStream | None:
event_type: Final = chunk.get("type")
event_name: Final = (
event_type.value
if isinstance(event_type, ResponsesAPIStreamEvents)
else event_type
if isinstance(event_type, str)
else None
)
if event_name != ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE:
return None
item: Final = as_str_mapping(chunk.get("item"))
if item is None or item.get("type") != "reasoning":
return None
output_index: Final = _stream_output_index(chunk)
signed_blocks: Final = _reasoning_done_thinking_blocks(item, output_index, output_index in summary_streamed_outputs)
if not signed_blocks:
return None
return ModelResponseStream(
choices=[
StreamingChoices(
index=0,
delta=Delta(thinking_blocks=list(signed_blocks)),
finish_reason=None,
)
]
)
def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
shifted: Final = {
key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
@ -1280,6 +1361,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
structured_output: bool = False,
replay_openai_history: bool = False,
completed: bool = False,
keeps_every_reasoning_item: bool = False,
) -> list["Choices"]:
"""
Convert Responses API output items to chat completion choices.
@ -1291,6 +1373,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
first message item becomes content
replay_openai_history: Record the output's item sequence for the next turn
completed: The response finished, so its item sequence is whole
keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last
Returns:
List of Choices objects
@ -1323,7 +1406,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
for item in output_items:
if isinstance(item, ResponseReasoningItem):
pending_reasoning_items = (
*(pending_reasoning_items if replay_openai_history else ()),
*(pending_reasoning_items if replay_openai_history or keeps_every_reasoning_item else ()),
_build_reasoning_item(
item_id=item.id,
encrypted_content=getattr(item, "encrypted_content", None),
@ -1573,6 +1656,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
== "openai",
completed=raw_response.status == "completed",
keeps_every_reasoning_item=serves_anthropic_messages(logging_obj),
)
response_is_incomplete: Final = raw_response.status == "incomplete" or (
@ -1637,6 +1721,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
json_mode: bool | None = False,
structured_output: bool = False,
replays_openai_history: bool = False,
streams_reasoning_signatures: bool = False,
) -> BaseModelResponseIterator:
return OpenAiResponsesToChatCompletionStreamIterator(
streaming_response,
@ -1644,6 +1729,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
json_mode,
structured_output=structured_output,
replays_openai_history=replays_openai_history,
streams_reasoning_signatures=streams_reasoning_signatures,
)
def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@ -2037,15 +2123,18 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
json_mode: bool | None = False,
structured_output: bool = False,
replays_openai_history: bool = False,
streams_reasoning_signatures: bool = False,
):
super().__init__(streaming_response, sync_stream, json_mode)
self._replays_openai_history: Final = replays_openai_history
self._streams_reasoning_signatures: Final = streams_reasoning_signatures
self._chat_completion_id: str | None = None
self._served_service_tier: str | None = None
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
self._structured_output: Final = structured_output
self._first_message_item_done: bool = False
self._reasoning_summary_streamed: bool = False
self._summary_streamed_outputs: frozenset[int | None] = frozenset()
def _handle_string_chunk(
self, str_line: Union[str, "BaseModel"]
@ -2263,7 +2352,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
]
)
elif event_type == "response.output_text.delta":
# Content part added to output
content_part = parsed_chunk.get("delta", None)
@ -2380,9 +2468,16 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
"""
verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk)
self._remember_served_service_tier(chunk)
reasoning_chunk: Final = (
_reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs)
if self._streams_reasoning_signatures
else None
)
translated: Final = (
self._empty_content_chunk()
if self._is_text_delta_after_structured_answer(chunk)
else reasoning_chunk
if reasoning_chunk is not None
else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
chunk,
tool_call_index_map=self._tool_call_index_map,
@ -2395,6 +2490,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._first_message_item_done = True
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
self._reasoning_summary_streamed = True
self._summary_streamed_outputs = self._summary_streamed_outputs | {_stream_output_index(chunk)}
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -194,11 +194,7 @@ def get_llm_provider(
resolved_model, provider, dynamic_api_key, resolved_api_base = _get_llm_provider(
model, custom_llm_provider, api_base, api_key, litellm_params
)
explicit_provider: Final = custom_llm_provider or (litellm_params.custom_llm_provider if litellm_params else None)
target: Final = (
resolve_model_latest_alias(resolved_model, provider) if explicit_provider or "/" in model else resolved_model
)
return target, provider, dynamic_api_key, resolved_api_base
return resolve_model_latest_alias(resolved_model, provider), provider, dynamic_api_key, resolved_api_base
def _get_llm_provider(

37
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -580,21 +580,25 @@ def _alias_groups(alias_owners: Mapping[str, str]) -> Mapping[str, tuple[str, ..
def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]:
"""
Expand ``aliases`` lists in model cost entries into top-level entries.
Expand alias rows into top-level entries that share their owner's ``AliasedCostEntry``.
Each alias gets a reference to the **same** ``AliasedCostEntry`` as the
canonical entry, which carries every name it is served under. The
``aliases`` key is removed from the entry so downstream code never sees it.
A row whose ``alias_of`` names another row is served as that row, and an
``aliases`` list on a row adds further names for it. Every name in a group
references the **same** ``AliasedCostEntry``, which carries every name it is
served under, and neither key survives into the served entries.
If an alias collides with an existing canonical entry the alias is
skipped and a warning is logged.
An alias that collides with a canonical entry or another alias, or a row whose
``alias_of`` names nothing, is skipped and a warning is logged.
"""
pointer_rows: Final = {
name: target for name, info in model_cost.items() if isinstance(target := info.get("alias_of"), str)
}
alias_owners: Final[dict[str, str]] = {}
keys_with_aliases: Final[list[str]] = []
for model_name, model_info in model_cost.items():
aliases: object = model_info.get("aliases")
if aliases is None:
if aliases is None or model_name in pointer_rows:
continue
keys_with_aliases.append(model_name)
if not isinstance(aliases, list):
@ -607,7 +611,7 @@ def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str,
if not aliases:
continue
for alias in aliases:
if alias in model_cost:
if alias in model_cost and alias not in pointer_rows:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (from '%s') "
"already exists as a canonical entry — skipping.",
@ -625,9 +629,26 @@ def _expand_model_aliases(model_cost: dict[str, dict[str, object]]) -> dict[str,
continue
alias_owners[alias] = model_name
for alias, target in pointer_rows.items():
if target not in model_cost or target in pointer_rows:
verbose_logger.warning(
"LiteLLM model alias '%s' points at '%s', which is not a model entry, dropping it.", alias, target
)
elif alias in alias_owners:
verbose_logger.warning(
"LiteLLM model alias conflict: alias '%s' (pointing at '%s') "
"was already claimed by another entry — skipping.",
alias,
target,
)
else:
alias_owners[alias] = target
# Remove the ``aliases`` key from entries so it doesn't pollute model info
for key in keys_with_aliases:
model_cost[key].pop("aliases", None)
for alias in pointer_rows.keys() - alias_owners.keys():
model_cost.pop(alias)
for owner, names in _alias_groups(alias_owners).items():
model_cost.update(dict.fromkeys(names, AliasedCostEntry(model_cost[owner], names)))

72
litellm/litellm_core_utils/model_latest_aliases.py
View file

@ -1,33 +1,75 @@
from collections.abc import Mapping
from functools import cache
from importlib.resources import files
from types import MappingProxyType
from typing import Final
from typing import Final, cast # noqa: TID251 # litellm.model_cost is an untyped module global
from pydantic import TypeAdapter
_ALIASES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]])
import litellm
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
def _cost_map() -> Mapping[str, object]:
return cast("Mapping[str, object]", litellm.model_cost) # cast-ok: per-lookup catalog validation is too slow
def _alias_entry(key: str) -> AliasedCostEntry | None:
entry: Final = _cost_map().get(key)
return entry if isinstance(entry, AliasedCostEntry) and entry.names[:1] not in ((), (key,)) else None
def _alias_provider(key: str, entry: AliasedCostEntry) -> str | None:
"""The provider an alias is requested through: its key prefix for a hosted alias such as
``bedrock/claude-haiku-latest`` (whose target row belongs to ``bedrock_converse``), else the target's provider"""
prefix, separator, _ = key.partition("/")
if separator and prefix in litellm.provider_list:
return prefix
provider: Final = entry.get("litellm_provider")
return provider if isinstance(provider, str) else None
def _alias_target(key: str, provider: str) -> str | None:
entry: Final = _alias_entry(key)
if entry is None or _alias_provider(key, entry) != provider:
return None
return entry.names[0].removeprefix(f"{provider}/")
@cache
def get_model_latest_aliases() -> Mapping[str, Mapping[str, str]]:
aliases: Final = _ALIASES_ADAPTER.validate_json(files("litellm").joinpath("model_latest_aliases.json").read_bytes())
return MappingProxyType({provider: MappingProxyType(models) for provider, models in aliases.items()})
"""``{provider: {alias: target}}`` for every alias row in the cost map, with provider prefixes stripped"""
rows: Final = tuple(
(provider, key, entry.names[0])
for key in _cost_map()
if (entry := _alias_entry(key)) is not None and (provider := _alias_provider(key, entry)) is not None
)
return MappingProxyType(
{
provider: MappingProxyType(
{
alias.removeprefix(f"{provider}/"): target.removeprefix(f"{provider}/")
for row_provider, alias, target in rows
if row_provider == provider
}
)
for provider in dict.fromkeys(provider for provider, _, _ in rows)
}
)
def resolve_model_latest_alias(model: str, provider: str) -> str:
aliases: Final = get_model_latest_aliases().get(provider)
return aliases.get(model, model) if aliases is not None else model
return _alias_target(f"{provider}/{model}", provider) or _alias_target(model, provider) or model
def expand_model_latest_alias(model: str) -> tuple[str, ...]:
provider, separator, name = model.partition("/")
target: Final = resolve_model_latest_alias(name, provider) if separator else name
return (model, f"{provider}/{target}") if separator and target != name else (model,)
if not separator:
entry: Final = _alias_entry(model)
return (model,) if entry is None else (model, entry.names[0])
target: Final = resolve_model_latest_alias(name, provider)
return (model, f"{provider}/{target}") if target != name else (model,)
def is_model_latest_alias(model: str, custom_llm_provider: str | None = None) -> bool:
prefix, separator, name = model.partition("/")
provider: Final = custom_llm_provider or prefix
provider: Final = custom_llm_provider or (prefix if separator else None)
if provider is None:
return _alias_entry(model) is not None
candidate: Final = name if separator and prefix == provider else model
return bool(custom_llm_provider or separator) and resolve_model_latest_alias(candidate, provider) != candidate
return resolve_model_latest_alias(candidate, provider) != candidate

95
litellm/litellm_core_utils/prompt_templates/reasoning_utils.py
View file

@ -5,6 +5,9 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from itertools import groupby
from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.types.llms.openai import (
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
@ -94,6 +97,90 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
_REASONING_SUMMARY_SEPARATOR: Final = "\n\n"
_OBJECT_TUPLE: Final = TypeAdapter(tuple[object, ...])
def _object_sequence(value: object) -> tuple[object, ...]:
if not isinstance(value, Sequence) or isinstance(value, (str, bytes)):
return ()
return _OBJECT_TUPLE.validate_python(value)
def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]:
return tuple(mapping for item in _object_sequence(value) if (mapping := as_str_mapping(item)) is not None)
def _reasoning_summary_part_text(part: object) -> str:
mapping: Final = as_str_mapping(part)
if mapping is not None:
text: Final = mapping.get("text")
return text if isinstance(text, str) else ""
text_attr: Final = getattr(part, "text", None)
return text_attr if isinstance(text_attr, str) else ""
def _reasoning_item_summary_text(item: Mapping[str, object]) -> str:
parts: Final = tuple(_reasoning_summary_part_text(part) for part in _object_sequence(item.get("summary")))
return _REASONING_SUMMARY_SEPARATOR.join(part for part in parts if part)
def thinking_block_from_reasoning_item(
item: object,
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None:
"""The chat thinking block that carries one Responses `reasoning` item back to an Anthropic client.
`encrypted_content` rides the same `litellm_encrypted_reasoning:` signature the Responses
adapter uses. A summary becomes thinking text. Bytes with no summary become `redacted_thinking`,
because an empty thinking string is not something the client can replay.
"""
mapping: Final = as_str_mapping(item)
if mapping is None or mapping.get("type") not in (None, "reasoning"):
return None
encrypted_content: Final = mapping.get("encrypted_content")
if not isinstance(encrypted_content, str) or not encrypted_content:
return None
signature: Final = encrypted_reasoning_signature(encrypted_content)
text: Final = _reasoning_item_summary_text(mapping)
if not text:
return ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=signature)
return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature)
def thinking_blocks_from_reasoning_items(
items: object,
) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]:
return tuple(
block for item in _object_sequence(items) if (block := thinking_block_from_reasoning_item(item)) is not None
)
def _block_encrypted_content(block: object) -> str | None:
mapping: Final = as_str_mapping(block)
if mapping is None:
return None
return encrypted_content_of_block(mapping)
def thinking_blocks_with_encrypted_reasoning(
thinking_blocks: object,
reasoning_items: object,
) -> tuple[Mapping[str, object], ...]:
"""Thinking blocks already on the message, plus any encrypted reasoning item they do not already carry."""
direct: Final = _iter_mappings(thinking_blocks)
represented: Final = frozenset(
content for block in direct if (content := _block_encrypted_content(block)) is not None
)
extra: Final = tuple(
block
for block in thinking_blocks_from_reasoning_items(_iter_mappings(reasoning_items))
if _block_encrypted_content(block) not in represented
)
return (*extra, *direct)
def is_encrypted_reasoning_block(block: object) -> bool:
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
@ -195,6 +282,14 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
)
def _content_carries_encrypted_reasoning(content: object) -> bool:
return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content))
def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool:
return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages))
def _strip_encrypted_reasoning_from_blocks(
content: object,
*,

1
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1176,6 +1176,7 @@ class CustomStreamWrapper:
and model_response.choices[0].delta.role is not None
)
or (getattr(model_response.choices[0].delta, "reasoning_items", None) is not None)
or getattr(model_response.choices[0].delta, "thinking_blocks", None)
):
return True
else:

16
litellm/llms/anthropic/chat/transformation.py
View file

@ -1750,7 +1750,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
llm_provider: str = "anthropic",
) -> AnthropicThinkingParam | None:
"""Capability probes read the cost map under ``custom_llm_provider``; ``llm_provider`` only tags raised exceptions."""
if reasoning_effort is None or reasoning_effort == "none":
if reasoning_effort is None:
return None
if reasoning_effort == "none":
if AnthropicModelInfo.thinks_by_default(
model, custom_llm_provider
) and not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider):
return AnthropicThinkingParam(type="disabled")
return None
if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
# without display, Anthropic defaults adaptive thinking to
@ -2158,12 +2164,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self._resolved_provider,
llm_provider=self._resolved_provider,
)
if mapped_thinking is None:
if mapped_thinking is None or mapped_thinking.get("type") == "disabled":
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
else:
if mapped_thinking is not None:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model, self._resolved_provider):
if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model(
model, self._resolved_provider
):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(effort_value)
if mapped_effort is None:
AnthropicConfig._raise_invalid_reasoning_effort(

230
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -1,5 +1,8 @@
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
import re
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from functools import partial
from typing import (
TYPE_CHECKING,
Final,
@ -11,17 +14,25 @@ from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm._internal_context import resent_on
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
is_encrypted_reasoning_block,
thinking_blocks_with_encrypted_reasoning,
)
from litellm.litellm_core_utils.tool_fields import (
ROUTES_WITHOUT_MCP_SERVERS,
drops_mcp_servers,
tools_without_mcp_servers,
)
from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages
from litellm.llms.anthropic.messages.adapters.transformation import (
AnthropicAdapter,
openai_thinking_routes_to_responses,
)
from litellm.llms.anthropic.messages.context_management import (
AnthropicContextManagementError,
@ -50,9 +61,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
)
from litellm.types.llms.openai import OpenAIWebSearchOptions
from litellm.types.utils import ModelResponse
from litellm.utils import get_model_info
if TYPE_CHECKING:
from litellm.chat_completions.dispatch import ChatResult
from litellm.router import Router
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
@ -70,22 +81,157 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
custom_llm_provider: str
_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE)
_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]"
@dataclass(frozen=True, slots=True)
class _ChatTargetReasoning:
effort: str
summary: str | None = None
def _provider_error_code(body: object) -> str | None:
mapping: Final = as_str_mapping(body)
if mapping is None:
return None
nested: Final = as_str_mapping(mapping.get("error"))
source: Final = nested if nested is not None else mapping
code: Final = source.get("code")
return code if isinstance(code, str) else None
def _logged_rejection_code(error: litellm.BadRequestError) -> str:
if error.code == "invalid_encrypted_content" or _provider_error_code(error.body) == "invalid_encrypted_content":
return "invalid_encrypted_content"
if "invalid_encrypted_content" in str(error):
return "invalid_encrypted_content"
return "could_not_be_verified"
def _completion_text(completion_kwargs: _CompletionKwargs, key: str) -> str:
value: Final = completion_kwargs.get(key)
return value if isinstance(value, str) else ""
def _message_carries_encrypted_reasoning(message: object) -> bool:
mapping: Final = as_str_mapping(message)
if mapping is None:
return False
blocks: Final = thinking_blocks_with_encrypted_reasoning(
mapping.get("thinking_blocks"), mapping.get("reasoning_items")
)
return any(is_encrypted_reasoning_block(block) for block in blocks)
def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool:
messages: Final = as_object_list(completion_kwargs.get("messages")) or ()
return any(_message_carries_encrypted_reasoning(message) for message in messages)
def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool:
error_message: Final = str(error)
return (
"invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None
)
def _is_rejected_reasoning(error: Exception) -> bool:
return isinstance(error, litellm.BadRequestError) and _rejects_encrypted_reasoning(error)
def resent_without_rejected_reasoning(sent_encrypted_reasoning: bool) -> AbstractContextManager[None]:
return resent_on(_is_rejected_reasoning) if sent_encrypted_reasoning else nullcontext()
def messages_without_rejected_reasoning(
messages: Sequence[Mapping[str, object]],
error: litellm.BadRequestError,
*,
sent_encrypted_reasoning: bool,
provider: str,
model: str,
) -> "_AnthropicMessages | None":
if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error):
return None
verbose_logger.warning(
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=%s model=%s status=%s code=%s",
provider,
model,
error.status_code,
_logged_rejection_code(error),
)
return _ANTHROPIC_MESSAGES.validate_python(
strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages])
)
def _messages_without_rejected_reasoning(
messages: _AnthropicMessages,
error: litellm.BadRequestError,
completion_kwargs: _CompletionKwargs,
sent_encrypted_reasoning: bool,
) -> "_AnthropicMessages | None":
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=sent_encrypted_reasoning,
provider=_completion_text(completion_kwargs, "custom_llm_provider"),
model=_completion_text(completion_kwargs, "model"),
)
async def _acompletion_without_rejected_reasoning(
completion_kwargs: _CompletionKwargs,
messages: _AnthropicMessages,
prepare_replay: _PrepareReplay,
) -> tuple[_CompletionKwargs, "ChatResult"]:
sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs)
try:
with resent_without_rejected_reasoning(sent_encrypted_reasoning):
return completion_kwargs, await litellm.acompletion(**completion_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(
messages, error, completion_kwargs, sent_encrypted_reasoning
)
if replayed_messages is None:
raise
replay_kwargs: Final = prepare_replay(replayed_messages)
return replay_kwargs, await litellm.acompletion(**replay_kwargs)
def _completion_without_rejected_reasoning(
completion_kwargs: _CompletionKwargs,
messages: _AnthropicMessages,
prepare_replay: _PrepareReplay,
) -> tuple[_CompletionKwargs, "ChatResult | Coroutine[object, object, ChatResult]"]:
sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(completion_kwargs)
try:
with resent_without_rejected_reasoning(sent_encrypted_reasoning):
return completion_kwargs, litellm.completion(**completion_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(
messages, error, completion_kwargs, sent_encrypted_reasoning
)
if replayed_messages is None:
raise
replay_kwargs: Final = prepare_replay(replayed_messages)
return replay_kwargs, litellm.completion(**replay_kwargs)
def _is_compaction_block(block: object) -> bool:
mapping: Final = as_str_mapping(block)
return mapping is not None and mapping.get("type") == "compaction"
def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
"""Return True when any message carries a ``compaction`` content block."""
for msg in messages:
content = msg.get("content")
if not isinstance(content, list):
continue
for block in content:
if isinstance(block, dict) and block.get("type") == "compaction":
return True
return False
return any(
any(map(_is_compaction_block, content)) for msg in messages if isinstance(content := msg.get("content"), list)
)
def _proxy_router_fallback() -> "Router | None":
@ -419,34 +565,17 @@ class LiteLLMMessagesToCompletionTransformationHandler:
If the user provides a `summary` field in the thinking dict, it is passed
through to the OpenAI reasoning params (opt-in per OpenAI spec).
"""
custom_llm_provider = completion_kwargs.get("custom_llm_provider")
if custom_llm_provider is None:
try:
_, inferred_provider, _, _ = litellm.utils.get_llm_provider(
model=cast(str, completion_kwargs.get("model"))
)
custom_llm_provider = inferred_provider
except Exception:
custom_llm_provider = None
if custom_llm_provider != "openai":
return
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
return
model: Final = completion_kwargs.get("model")
try:
model_info: Final = get_model_info(model=cast(str, model), custom_llm_provider=custom_llm_provider)
if model_info and model_info.get("supports_reasoning") is False:
# Model doesn't support reasoning/responses API, don't route
return
except Exception:
pass
custom_llm_provider: Final = completion_kwargs.get("custom_llm_provider")
if not isinstance(model, str) or thinking is None:
return
if not openai_thinking_routes_to_responses(
model, custom_llm_provider if isinstance(custom_llm_provider, str) else None, thinking
):
return
if isinstance(model, str) and model and "responses/" not in model:
local_model: Final = model.removeprefix(f"{custom_llm_provider}/")
completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}"
if "responses/" not in model:
completion_kwargs["model"] = f"openai/responses/{model.removeprefix('openai/')}"
auto_summary: Final = is_reasoning_auto_summary_enabled()
@ -677,6 +806,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
custom_llm_provider=target_provider,
api_base=api_base if isinstance(api_base, str) else None,
deployment_model_info=as_str_mapping(provider_kwargs.get("model_info")),
reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs),
drop_params=drop_params,
)
@ -804,12 +934,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
parent_trace_id=sub_call_trace_id,
)
(
completion_kwargs,
tool_name_mapping,
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
prepare: Final = partial(
LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs,
max_tokens=max_tokens,
messages=effective_messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
@ -824,8 +951,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
completion_response: Final = await litellm.acompletion(**completion_kwargs)
initial_kwargs, tool_name_mapping = prepare(messages=effective_messages)
completion_kwargs, completion_response = await _acompletion_without_rejected_reasoning(
initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0]
)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
if stream:
@ -951,12 +1080,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
effective_system: Final = polyfill_result.system if polyfill_result is not None else system
(
completion_kwargs,
tool_name_mapping,
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
prepare: Final = partial(
LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs,
max_tokens=max_tokens,
messages=effective_messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
@ -971,8 +1097,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
completion_response: Final = litellm.completion(**completion_kwargs)
initial_kwargs, tool_name_mapping = prepare(messages=effective_messages)
completion_kwargs, completion_response = _completion_without_rejected_reasoning(
initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0]
)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
if stream:

27
litellm/llms/anthropic/messages/adapters/streaming_iterator.py
View file

@ -132,14 +132,21 @@ def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
assert_never(delta_type)
def _first_thinking_block_index(chunk: "ModelResponseStream") -> int | None:
def _first_thinking_block(chunk: "ModelResponseStream") -> Mapping[str, object] | None:
delta: Final = _optional_attr(chunk.choices[0], "delta") if chunk.choices else None
blocks: Final = _optional_attr_sequence(delta, "thinking_blocks")
first_block: Final = as_str_mapping(blocks[0]) if blocks else None
index: Final = first_block.get("index") if first_block is not None else None
return as_str_mapping(blocks[0]) if blocks else None
def _thinking_block_index(block: Mapping[str, object] | None) -> int | None:
index: Final = block.get("index") if block is not None else None
return index if isinstance(index, int) else None
def _closes_thinking_block(block: Mapping[str, object] | None) -> bool:
return block is not None and _thinking_block_index(block) is not None and bool(block.get("signature"))
def _optional_str_attr(obj: object, name: str) -> str | None:
value: Final = _optional_attr(obj, name)
return value if isinstance(value, str) else None
@ -632,6 +639,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
sent_content_block_finish: bool = False
current_content_block_type: Literal["text", "tool_use", "thinking", "redacted_thinking"] = "text"
current_thinking_block_index: int | None = None
current_thinking_block_closed: bool = False
sent_last_message: bool = False
holding_chunk: ContentBlockDelta | None = None
holding_stop_reason_chunk: MessageBlockDelta | None = None
@ -1575,9 +1583,16 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
original_name: Final = self.tool_name_mapping.get(truncated_name, truncated_name)
tool_block["name"] = original_name
thinking_block_index: Final = _first_thinking_block_index(chunk)
thinking_block: Final = _first_thinking_block(chunk)
thinking_block_index: Final = _thinking_block_index(thinking_block)
previous_thinking_block_index: Final = self.current_thinking_block_index
if thinking_block_index is not None or block_type != self.current_content_block_type:
follows_closed_thinking_block: Final = self.current_thinking_block_closed and block_type == "thinking"
self.current_thinking_block_closed = _closes_thinking_block(thinking_block)
if (
thinking_block_index is not None
or block_type != self.current_content_block_type
or follows_closed_thinking_block
):
self.current_thinking_block_index = thinking_block_index
if block_type != self.current_content_block_type:
@ -1585,7 +1600,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
self.current_content_block_start = content_block_start
return True
if (
if follows_closed_thinking_block or (
thinking_block_index is not None
and previous_thinking_block_index is not None
and thinking_block_index != previous_thinking_block_index

180
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -2,13 +2,19 @@ import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, require_attributes
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_items,
normalize_drop_params,
require_attributes,
)
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.messages.utils import (
@ -153,6 +159,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
reasoning_content_from_thinking_blocks,
thinking_blocks_with_encrypted_reasoning,
)
from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import (
parse_tool_call_arguments,
@ -263,27 +270,72 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
def _is_azure_responses_target(
@dataclass(frozen=True, slots=True)
class ChatRouteInputs:
tools: Sequence[Mapping[str, object]] | None = None
reasoning_effort: str | Mapping[str, object] | None = None
api_base: str | None = None
thinking: Mapping[str, object] | None = None
reasoning_summary: object | None = None
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
def _inferred_provider(model: str) -> str | None:
try:
return litellm.utils.get_llm_provider(model=model)[1]
except Exception:
return None
def openai_thinking_routes_to_responses(
model: str | None,
custom_llm_provider: str | None,
thinking: Mapping[str, object] | None,
) -> bool:
if not model or thinking is None or thinking.get("type") != "enabled":
return False
provider: Final = custom_llm_provider if custom_llm_provider is not None else _inferred_provider(model)
if provider != "openai":
return False
try:
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider=provider)
except Exception:
return True
return model_info.get("supports_reasoning") is not False
def _keeps_encrypted_reasoning_for_responses(
model: str | None,
custom_llm_provider: str | None,
deployment_model_info: Mapping[str, object] | None,
route: ChatRouteInputs,
) -> bool:
if not model:
return False
if custom_llm_provider not in (None, LlmProviders.AZURE.value):
return False
if custom_llm_provider is None and not model.startswith("azure/"):
return False
from litellm.responses.bridge_check import chat_completion_ends_in_responses_api
from litellm.responses.bridge_check import responses_api_bridge_check
resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
model_info, _ = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
deployment_model_info=deployment_model_info,
return openai_thinking_routes_to_responses(
model, custom_llm_provider, route.thinking
) or chat_completion_ends_in_responses_api(
model,
custom_llm_provider,
deployment_model_info,
tools=route.tools,
reasoning_effort=route.reasoning_effort,
reasoning_summary=route.reasoning_summary,
api_base=route.api_base,
)
return model_info.get("mode") == "responses"
def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]:
return thinking_blocks_with_encrypted_reasoning(
getattr(source, "thinking_blocks", None),
getattr(source, "reasoning_items", None),
)
def _streamed_thinking_blocks(delta: object) -> tuple[Mapping[str, object], ...]:
return tuple(block for item in attribute_items(delta, "thinking_blocks") if (block := as_str_mapping(item)))
def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam:
@ -322,6 +374,7 @@ class AnthropicAdapter:
custom_llm_provider: str | None = None,
api_base: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
reasoning_summary: object | None = None,
drop_params: bool = True,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
@ -358,6 +411,7 @@ class AnthropicAdapter:
custom_llm_provider=custom_llm_provider,
api_base=api_base,
deployment_model_info=deployment_model_info,
reasoning_summary=reasoning_summary,
drop_params=drop_params,
report_reasoning_drops=True,
)
@ -556,11 +610,12 @@ class LiteLLMAnthropicMessagesAdapter:
custom_llm_provider: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
preserve_midturn_system: bool = False,
route: ChatRouteInputs = _NO_ROUTE_INPUTS,
) -> list[AllMessageValues]:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = (
messages
if _is_azure_responses_target(model, custom_llm_provider, deployment_model_info)
if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider, deployment_model_info, route)
else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
)
leading_count: Final = next(
@ -1343,6 +1398,7 @@ class LiteLLMAnthropicMessagesAdapter:
custom_llm_provider: str | None = None,
api_base: str | None = None,
deployment_model_info: Mapping[str, object] | None = None,
reasoning_summary: object | None = None,
preserve_midturn_system: bool = False,
drop_params: bool = True,
report_reasoning_drops: bool = False,
@ -1356,27 +1412,15 @@ class LiteLLMAnthropicMessagesAdapter:
for tools that exceeded OpenAI's 64-char limit
"""
# Debug: Processing Anthropic message request
new_messages: list[AllMessageValues] = []
tool_name_mapping: dict[str, str] = {}
## CONVERT ANTHROPIC MESSAGES TO OPENAI
messages_list: Final[list[AllAnthropicPassThroughMessageValues]] = cast(
list[AllAnthropicPassThroughMessageValues],
anthropic_message_request["messages"],
)
new_messages = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
preserve_midturn_system=preserve_midturn_system,
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)
new_kwargs: Final[ChatCompletionRequest] = {
"model": anthropic_message_request["model"],
"messages": new_messages,
"messages": [],
}
## CONVERT METADATA (user_id + litellm metadata)
self._translate_metadata_to_openai(
@ -1403,6 +1447,24 @@ class LiteLLMAnthropicMessagesAdapter:
drop_params=drop_params,
report_reasoning_drops=report_reasoning_drops,
)
## CONVERT ANTHROPIC MESSAGES TO OPENAI, after tools and thinking pick the route
new_messages: Final = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
deployment_model_info=deployment_model_info,
preserve_midturn_system=preserve_midturn_system,
route=ChatRouteInputs(
tools=new_kwargs.get("tools"),
reasoning_effort=new_kwargs.get("reasoning_effort"),
api_base=api_base,
thinking=anthropic_message_request.get("thinking"),
reasoning_summary=reasoning_summary,
),
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)
new_kwargs["messages"] = new_messages
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(
anthropic_message_request=anthropic_message_request,
@ -1507,9 +1569,9 @@ class LiteLLMAnthropicMessagesAdapter:
for choice in choices
):
new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks)
# Handle thinking blocks first
if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks:
for thinking_block in choice.message.thinking_blocks:
signed_thinking_blocks = _anthropic_thinking_blocks(choice.message)
if signed_thinking_blocks:
for thinking_block in signed_thinking_blocks:
if thinking_block.get("type") == "thinking":
if is_empty_unsigned_thinking_block(thinking_block):
continue
@ -1821,24 +1883,24 @@ class LiteLLMAnthropicMessagesAdapter:
choice.delta
) is not None:
return "text", TextBlock(type="text", text="")
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
thinking_blocks = choice.delta.thinking_blocks or []
if len(thinking_blocks) > 0:
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
type="redacted_thinking", data=thinking_block.get("data") or ""
)
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)):
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
redacted_data = thinking_block.get("data")
return "redacted_thinking", ChatCompletionRedactedThinkingBlock(
type="redacted_thinking",
data=redacted_data if isinstance(redacted_data, str) else "",
)
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
assert isinstance(thinking, str)
assert isinstance(signature, str)
assert isinstance(thinking, str)
assert isinstance(signature, str)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking if signature else "", signature=signature
)
return "thinking", ChatCompletionThinkingBlock(
type="thinking", thinking=thinking if signature else "", signature=signature
)
# OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning
# parsers) populate ``reasoning_content`` without ``thinking_blocks``.
# ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the
@ -1871,19 +1933,17 @@ class LiteLLMAnthropicMessagesAdapter:
for tool in choice.delta.tool_calls:
if tool.function is not None and tool.function.arguments is not None:
partial_json = (partial_json or "") + tool.function.arguments
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"):
thinking_blocks = choice.delta.thinking_blocks or []
if len(thinking_blocks) > 0:
for thinking_block in thinking_blocks:
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)):
for thinking_block in thinking_blocks:
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""
signature = thinking_block.get("signature") or ""
assert isinstance(thinking, str)
assert isinstance(signature, str)
assert isinstance(thinking, str)
assert isinstance(signature, str)
reasoning_content += thinking
reasoning_signature += signature
reasoning_content += thinking
reasoning_signature += signature
# Handle reasoning_content when thinking_blocks is not present
# This handles providers like OpenRouter that return reasoning_content
elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"):

12
litellm/llms/anthropic/messages/native/transformation.py
View file

@ -471,7 +471,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
"""Map OpenAI-style ``reasoning_effort`` to native Anthropic params.
Caller-supplied ``thinking`` / ``output_config`` win over the alias.
``effort='none'`` clears both. Invalid efforts raise a 400. A mapped
``effort='none'`` disables optional default thinking and clears effort. Invalid efforts raise a 400. A mapped
thinking budget is capped below ``max_tokens`` and dropped when even
the minimum budget cannot fit.
"""
@ -485,6 +485,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
if not isinstance(reasoning_effort, str):
return
if reasoning_effort == "none":
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
try:
mapped_thinking: Final = AnthropicConfig.map_reasoning_effort(
reasoning_effort=reasoning_effort,
@ -495,8 +499,6 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raise AnthropicError(message=str(e.message), status_code=400)
if mapped_thinking is None:
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
return
fitted_thinking: Final = AnthropicConfig.cap_thinking_budget_to_max_tokens(mapped_thinking, max_tokens)
@ -505,7 +507,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
return
optional_params.setdefault("thinking", fitted_thinking)
if AnthropicModelInfo.is_adaptive_thinking_model(model, custom_llm_provider):
if fitted_thinking.get("type") != "disabled" and AnthropicModelInfo.is_adaptive_thinking_model(
model, custom_llm_provider
):
mapped_effort: Final = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
raise AnthropicError(

168
litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -19,13 +19,14 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_with_readable_mode,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -43,7 +44,12 @@ from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..adapters.handler import (
context_management_explicitly_dropped,
messages_without_rejected_reasoning,
request_safeguards_evaluator,
resent_without_rejected_reasoning,
)
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
@ -391,6 +397,76 @@ def _build_responses_kwargs(
return responses_kwargs
_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]]
_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator
_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult]
def _is_encrypted_reasoning_item(item: object) -> bool:
mapping: Final = as_str_mapping(item)
return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content"))
def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool:
return any(map(_is_encrypted_reasoning_item, as_object_list(responses_kwargs.get("input")) or ()))
def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str:
value: Final = responses_kwargs.get(key)
return value if isinstance(value, str) else ""
def _messages_without_rejected_reasoning(
messages: AnthropicRequestMessages,
error: litellm.BadRequestError,
responses_kwargs: Mapping[str, object],
sent_encrypted_reasoning: bool,
) -> AnthropicRequestMessages | None:
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=sent_encrypted_reasoning,
provider=_kwarg_text(responses_kwargs, "custom_llm_provider"),
model=_kwarg_text(responses_kwargs, "model"),
)
async def _aresponses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _ResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs)
try:
with resent_without_rejected_reasoning(sent_encrypted_reasoning):
return responses_kwargs, await litellm.aresponses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(
messages, error, responses_kwargs, sent_encrypted_reasoning
)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, await litellm.aresponses(**replay_kwargs)
def _responses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _SyncResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
sent_encrypted_reasoning: Final = _sends_encrypted_reasoning(responses_kwargs)
try:
with resent_without_rejected_reasoning(sent_encrypted_reasoning):
return responses_kwargs, litellm.responses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(
messages, error, responses_kwargs, sent_encrypted_reasoning
)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, litellm.responses(**replay_kwargs)
class LiteLLMMessagesToResponsesAPIHandler:
"""
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
@ -422,28 +498,32 @@ class LiteLLMMessagesToResponsesAPIHandler:
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
result: Final = await litellm.aresponses(**responses_kwargs)
responses_kwargs, result = await _aresponses_without_rejected_reasoning(
polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
@ -526,27 +606,31 @@ class LiteLLMMessagesToResponsesAPIHandler:
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
result: Final = litellm.responses(**responses_kwargs)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
responses_kwargs, result = _responses_without_rejected_reasoning(
polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(

8
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -583,12 +583,14 @@ class AmazonConverseConfig(BaseConfig):
custom_llm_provider="bedrock",
llm_provider="bedrock_converse",
)
if mapped_thinking is None:
if mapped_thinking is None or mapped_thinking.get("type") == "disabled":
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
else:
if mapped_thinking is not None:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig.is_adaptive_thinking_model(model, "bedrock"):
if mapped_thinking.get("type") != "disabled" and AnthropicConfig.is_adaptive_thinking_model(
model, "bedrock"
):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort)
if mapped_effort is None:
AnthropicConfig.raise_invalid_reasoning_effort(

8
litellm/llms/databricks/chat/transformation.py
View file

@ -629,12 +629,14 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
custom_llm_provider="databricks",
llm_provider="databricks",
)
if mapped_thinking is None:
if mapped_thinking is None or mapped_thinking.get("type") == "disabled":
optional_params.pop("thinking", None)
optional_params.pop("output_config", None)
else:
if mapped_thinking is not None:
optional_params["thinking"] = mapped_thinking
if AnthropicConfig._is_adaptive_thinking_model(model, "databricks"):
if mapped_thinking.get("type") != "disabled" and AnthropicConfig._is_adaptive_thinking_model(
model, "databricks"
):
mapped_effort: str | None = None
if isinstance(reasoning_effort_value, str):
mapped_effort = REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT.get(reasoning_effort_value)

6
litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
View file

@ -166,7 +166,11 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
) -> tuple[dict[str, object], RequestFiles | None]:
inline_parts: Final = self._prepare_inline_image_parts(image) if image else []
if not inline_parts:
raise ValueError("Vertex AI Gemini image edit requires at least one image.")
raise litellm.BadRequestError(
message="Vertex AI Gemini image edit requires at least one image.",
model=model,
llm_provider="vertex_ai",
)
# Build parts list with image and prompt (if provided)
text_parts: Final[list[HttpxPartType]] = [{"text": prompt}] if prompt is not None and prompt != "" else []

33
litellm/main.py
View file

@ -28,6 +28,7 @@ from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
from litellm._internal_context import is_resent_failure
from litellm._logging import redact_string
from litellm._uuid import uuid
@ -6638,6 +6639,10 @@ def completion(
)
def _is_not_resent_failure(error: BaseException) -> bool:
return isinstance(error, Exception) and not is_resent_failure(error)
def completion_with_retries(*args, **kwargs):
"""
Executes a litellm.completion() with 3 retries
@ -6657,10 +6662,15 @@ def completion_with_retries(*args, **kwargs):
retryer = tenacity.Retrying(
wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
else:
retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True)
retryer = tenacity.Retrying(
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
return retryer(original_function, *args, **kwargs)
@ -6683,10 +6693,15 @@ async def acompletion_with_retries(*args, **kwargs):
retryer = tenacity.AsyncRetrying(
wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
else:
retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True)
retryer = tenacity.AsyncRetrying(
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
return await retryer(original_function, *args, **kwargs)
@ -6711,10 +6726,15 @@ def responses_with_retries(*args, **kwargs):
retryer = tenacity.Retrying(
wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
else:
retryer = tenacity.Retrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True)
retryer = tenacity.Retrying(
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
return retryer(original_function, *args, **kwargs)
@ -6738,10 +6758,15 @@ async def aresponses_with_retries(*args, **kwargs):
retryer = tenacity.AsyncRetrying(
wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
else:
retryer = tenacity.AsyncRetrying(stop=tenacity.stop_after_attempt(num_retries), reraise=True)
retryer = tenacity.AsyncRetrying(
stop=tenacity.stop_after_attempt(num_retries),
retry=tenacity.retry_if_exception(_is_not_resent_failure),
reraise=True,
)
return await retryer(original_function, *args, **kwargs)

21
litellm/model_latest_aliases.json
View file

@ -1,21 +0,0 @@
{
"openai": {
"gpt-luna-latest": "gpt-6-luna",
"gpt-sol-latest": "gpt-6.1-sol",
"gpt-terra-latest": "gpt-6.1-terra",
"gpt-astra-latest": "gpt-6-astra",
"gpt-mini-latest": "gpt-5.4-mini"
},
"anthropic": {
"claude-opus-latest": "claude-opus-5-5",
"claude-sonnet-latest": "claude-sonnet-5-5",
"claude-haiku-latest": "claude-haiku-5-5",
"claude-fable-latest": "claude-fable-5-1"
},
"xai": {"grok-latest": "grok-4.7"},
"moonshot": {"kimi-latest": "kimi-k3"},
"zai": {"glm-latest": "glm-5.3", "glm-flash-latest": "glm-5.3-flash"},
"deepseek": {"deepseek-pro-latest": "deepseek-v4-pro", "deepseek-flash-latest": "deepseek-v4-flash"},
"dashscope": {"qwen-max-latest": "qwen3.8-max", "qwen-flash-latest": "qwen3.8-flash"},
"minimax": {"minimax-latest": "MiniMax-M3"}
}

339
litellm/model_prices_and_context_window_backup.json
View file

@ -1969,6 +1969,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/claude-fable-latest": {
"alias_of": "global.anthropic.claude-fable-5-1"
},
"us.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 2.2e-05,
@ -2305,6 +2308,9 @@
"region": "us-east-1"
}
},
"bedrock/claude-opus-latest": {
"alias_of": "global.anthropic.claude-opus-5-5"
},
"us.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
@ -4063,6 +4069,9 @@
"deprecation_date": "2027-12-05",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
},
"azure_ai/claude-fable-latest": {
"alias_of": "azure_ai/claude-fable-5-1"
},
"azure_ai/claude-mythos-5-1": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
@ -4165,6 +4174,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01"
},
"azure_ai/claude-opus-latest": {
"alias_of": "azure_ai/claude-opus-5-5"
},
"azure_ai/claude-opus-4-8": {
"deprecation_date": "2027-09-01",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -4382,6 +4394,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-astra-latest": {
"alias_of": "azure_ai/gpt-6-astra"
},
"azure_ai/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -4430,6 +4445,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-luna-latest": {
"alias_of": "azure_ai/gpt-6-luna"
},
"azure_ai/gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -4526,6 +4544,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-sol-latest": {
"alias_of": "azure_ai/gpt-6.1-sol"
},
"azure_ai/gpt-6.1-terra": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
@ -4574,6 +4595,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-terra-latest": {
"alias_of": "azure_ai/gpt-6.1-terra"
},
"azure_ai/gpt-5.5": {
"deprecation_date": "2027-10-26",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -5036,6 +5060,9 @@
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-mini-latest": {
"alias_of": "azure_ai/gpt-5.4-mini"
},
"azure_ai/gpt-5.4-mini-2026-03-17": {
"deprecation_date": "2027-09-21",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -9080,6 +9107,9 @@
"cache_read_input_token_cost_flex": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-06
},
"azure/gpt-astra-latest": {
"alias_of": "azure/gpt-6-astra"
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
@ -9184,6 +9214,9 @@
"output_cost_per_token_above_272k_tokens_flex": 3.75e-07,
"output_cost_per_token_flex": 2.5e-07
},
"azure/gpt-luna-latest": {
"alias_of": "azure/gpt-6-luna"
},
"azure/gpt-6-luna-2026-09-22": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -9403,6 +9436,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-terra-latest": {
"alias_of": "azure/gpt-6.1-terra"
},
"azure/gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -9455,6 +9491,9 @@
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06
},
"azure/gpt-sol-latest": {
"alias_of": "azure/gpt-6.1-sol"
},
"azure/gpt-6.1-sol-2026-09-29": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -11146,6 +11185,9 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-mini-latest": {
"alias_of": "azure/gpt-5.4-mini"
},
"azure/gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -13212,6 +13254,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/kimi-latest": {
"alias_of": "azure_ai/FW-Kimi-K3"
},
"azure_ai/FW-MiniMax-M2.5": {
"deprecation_date": "2027-07-01",
"cache_read_input_token_cost": 3.3e-08,
@ -13251,6 +13296,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/minimax-latest": {
"alias_of": "azure_ai/FW-MiniMax-M3"
},
"azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 6e-08,
@ -13656,6 +13704,9 @@
"cache_read_input_token_cost": 1.45e-07,
"supports_prompt_caching": true
},
"azure_ai/deepseek-pro-latest": {
"alias_of": "azure_ai/deepseek-v4-pro"
},
"azure_ai/deepseek-v4-flash": {
"deprecation_date": "2028-02-20",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -13674,6 +13725,9 @@
"cache_read_input_token_cost": 2.8e-08,
"supports_prompt_caching": true
},
"azure_ai/deepseek-flash-latest": {
"alias_of": "azure_ai/deepseek-v4-flash"
},
"azure_ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 1.4e-08,
"deprecation_date": "2026-12-03",
@ -16623,6 +16677,9 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing",
"supports_web_search": true
},
"claude-sonnet-latest": {
"alias_of": "claude-sonnet-5-5"
},
"claude-sonnet-4-6": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17024,6 +17081,9 @@
"supports_native_structured_output": true,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-fable-latest": {
"alias_of": "claude-fable-5-1"
},
"claude-opus-5-5": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17069,6 +17129,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-latest": {
"alias_of": "claude-opus-5-5"
},
"claude-opus-5": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17539,6 +17602,8 @@
"supports_reasoning": true
},
"codestral/codestral-2405": {
"deprecation_date": "2025-06-16",
"deprecation_source": "https://docs.mistral.ai/getting-started/models/",
"input_cost_per_token": 0.0,
"litellm_provider": "codestral",
"max_input_tokens": 32000,
@ -18866,6 +18931,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen-max-latest": {
"alias_of": "dashscope/qwen3.8-max"
},
"dashscope/qwen3.8-flash": {
"cache_creation_input_token_cost": 2e-07,
"cache_read_input_token_cost": 1.6e-08,
@ -18885,6 +18953,9 @@
"supports_vision": true,
"supports_web_search": true
},
"dashscope/qwen-flash-latest": {
"alias_of": "dashscope/qwen3.8-flash"
},
"dashscope/qwen3.8-omni-flash": {
"cache_read_input_token_cost": 1.6e-08,
"input_cost_per_token": 1.5e-07,
@ -21765,6 +21836,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"databricks/claude-fable-latest": {
"alias_of": "databricks/databricks-claude-fable-5-1"
},
"databricks/databricks-claude-haiku-4-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_creation_input_token_cost_above_1hr": 1.99997e-06,
@ -22005,6 +22079,9 @@
"supports_xhigh_reasoning_effort": true,
"thinking_always_on": true
},
"databricks/claude-opus-latest": {
"alias_of": "databricks/databricks-claude-opus-5-5"
},
"databricks/databricks-claude-sonnet-4": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
@ -22722,6 +22799,9 @@
"cache_read_input_token_cost_priority": 4.55e-07,
"cache_creation_input_token_cost_priority": 2.45e-06
},
"databricks/glm-latest": {
"alias_of": "databricks/databricks-glm-5-3"
},
"databricks/databricks-glm-5-3-flash": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 3.003e-08,
@ -22755,6 +22835,9 @@
"cache_read_input_token_cost_priority": 5.25e-08,
"cache_creation_input_token_cost_priority": 2.625e-07
},
"databricks/glm-flash-latest": {
"alias_of": "databricks/databricks-glm-5-3-flash"
},
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
@ -22917,6 +23000,9 @@
"cache_read_input_token_cost_priority": 1.5001e-07,
"cache_creation_input_token_cost_priority": 1.50003e-06
},
"databricks/gpt-mini-latest": {
"alias_of": "databricks/databricks-gpt-5-4-mini"
},
"databricks/databricks-gpt-5-4-nano": {
"cache_creation_input_token_cost": 1.9999e-07,
"cache_read_input_token_cost": 2.002e-08,
@ -23313,6 +23399,9 @@
"cache_read_input_token_cost_priority": 5.25e-07,
"cache_creation_input_token_cost_priority": 5.25e-06
},
"databricks/kimi-latest": {
"alias_of": "databricks/databricks-kimi-k3"
},
"databricks/system.ai.kimi-k3": {
"litellm_provider": "databricks",
"mode": "chat",
@ -28841,6 +28930,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/minimax-latest": {
"alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3"
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
"litellm_provider": "fireworks_ai",
@ -29341,7 +29433,7 @@
"source": "https://api.friendli.ai/serverless/v1/models"
},
"ft:babbage-002": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 1.6e-06,
@ -29355,7 +29447,7 @@
"source": "https://developers.openai.com/api/docs/pricing"
},
"ft:davinci-002": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 1.2e-05,
@ -29414,7 +29506,7 @@
"supports_tool_choice": true
},
"ft:gpt-3.5-turbo-1106": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 3e-06,
@ -33787,6 +33879,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-fable-latest": {
"alias_of": "github_copilot/claude-fable-5.1"
},
"github_copilot/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_read_input_token_cost": 1e-07,
@ -33830,6 +33925,9 @@
"supports_vision": true,
"thinking_always_on": false
},
"github_copilot/claude-haiku-latest": {
"alias_of": "github_copilot/claude-haiku-5.5"
},
"github_copilot/claude-opus-4.8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
@ -33918,6 +34016,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-opus-latest": {
"alias_of": "github_copilot/claude-opus-5.5"
},
"github_copilot/claude-sonnet-5": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
@ -33961,6 +34062,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-sonnet-latest": {
"alias_of": "github_copilot/claude-sonnet-5.5"
},
"github_copilot/gemini-3.7-flash": {
"cache_read_input_token_cost": 7.5e-08,
"deprecation_date": "2026-10-19",
@ -34306,6 +34410,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-astra-latest": {
"alias_of": "github_copilot/gpt-6-astra"
},
"github_copilot/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -34329,6 +34436,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-luna-latest": {
"alias_of": "github_copilot/gpt-6-luna"
},
"github_copilot/gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -34375,6 +34485,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-sol-latest": {
"alias_of": "github_copilot/gpt-6.1-sol"
},
"github_copilot/kimi-k3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
@ -34392,6 +34505,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/kimi-latest": {
"alias_of": "github_copilot/kimi-k3"
},
"github_copilot/mai-code-1.1-flash": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,
@ -37685,6 +37801,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-astra-latest": {
"alias_of": "gpt-6-astra"
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
@ -37882,6 +38001,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-luna-latest": {
"alias_of": "gpt-6-luna"
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
@ -39065,6 +39187,9 @@
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"gpt-mini-latest": {
"alias_of": "gpt-5.4-mini"
},
"gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
@ -43023,6 +43148,9 @@
"max_input_tokens": 1000000,
"max_output_tokens": 128000
},
"minimax/minimax-latest": {
"alias_of": "minimax/MiniMax-M3"
},
"minimax/MiniMax-H3": {
"litellm_provider": "minimax",
"mode": "video_generation",
@ -44487,6 +44615,28 @@
"supports_response_schema": true,
"supports_vision": true
},
"moonshot/kimi-k3.1-pro-highspeed": {
"cache_creation_input_token_cost": 0.000004,
"cache_creation_input_token_cost_above_1hr": 0.000008,
"cache_read_input_token_cost": 2e-7,
"input_cost_per_token": 0.000004,
"litellm_provider": "moonshot",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 0.00002,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://platform.kimi.ai/docs/pricing/chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true
},
"moonshot/kimi-k3": {
"cache_creation_input_token_cost": 3e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
@ -44511,6 +44661,9 @@
"supports_video_input": true,
"supports_vision": true
},
"moonshot/kimi-latest": {
"alias_of": "moonshot/kimi-k3"
},
"moonshot/moonshot-v1-128k": {
"deprecation_date": "2026-08-31",
"input_cost_per_token": 2e-06,
@ -49673,6 +49826,8 @@
"mode": "search"
},
"text-completion-codestral/codestral-2405": {
"deprecation_date": "2025-06-16",
"deprecation_source": "https://docs.mistral.ai/getting-started/models/",
"input_cost_per_token": 0.0,
"litellm_provider": "text-completion-codestral",
"max_input_tokens": 32000,
@ -50021,13 +50176,13 @@
"supports_reasoning": true
},
"together_ai/Qwen/Qwen3.7-Max": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.5e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token": 7.5e-06,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
@ -50110,6 +50265,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-pro-latest": {
"alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"together_ai/meta-models/Muse-Glimmer-30B": {
"cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 3.5e-07,
@ -50156,6 +50314,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/kimi-latest": {
"alias_of": "together_ai/moonshotai/Kimi-K3"
},
"together_ai/thinkingmachines/Inkling": {
"cache_read_input_token_cost": 1.7e-07,
"deprecation_date": "2026-11-06",
@ -50206,6 +50367,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/glm-latest": {
"alias_of": "together_ai/zai-org/GLM-5.3"
},
"together_ai/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
@ -50223,6 +50387,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/glm-flash-latest": {
"alias_of": "together_ai/zai-org/GLM-5.3-Flash"
},
"tts-1": {
"deprecation_date": "2027-01-06",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
@ -52993,6 +53160,9 @@
"output_cost_per_token_batches": 2.5e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-fable-latest": {
"alias_of": "vertex_ai/claude-fable-5-1"
},
"vertex_ai/claude-fable-5@default": {
"deprecation_date": "2027-06-08",
"regional_endpoint_uplift_multiplier": 1.1,
@ -53191,6 +53361,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-opus-latest": {
"alias_of": "vertex_ai/claude-opus-5-5"
},
"vertex_ai/claude-opus-5-5@default": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
@ -55739,6 +55912,9 @@
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-latest": {
"alias_of": "xai/grok-4.7"
},
"zai.glm-4.7": {
"deprecation_date": "2027-03-30",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html",
@ -55840,6 +56016,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"zai/glm-latest": {
"alias_of": "zai/glm-5.3"
},
"zai/glm-5.3-flash": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-08,
@ -55857,6 +56036,9 @@
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true
},
"zai/glm-flash-latest": {
"alias_of": "zai/glm-5.3-flash"
},
"zai/glm-5v-turbo": {
"cache_read_input_token_cost": 2.4e-07,
"input_cost_per_token": 1.2e-06,
@ -61493,6 +61675,9 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html",
"supports_web_search": true
},
"bedrock_mantle/gpt-astra-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6-astra"
},
"bedrock_mantle/openai.gpt-6-sol": {
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
@ -61573,6 +61758,9 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
"supports_web_search": true
},
"bedrock_mantle/gpt-luna-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6-luna"
},
"us.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-05,
@ -61741,6 +61929,9 @@
"/v1/responses"
]
},
"bedrock/gpt-astra-latest": {
"alias_of": "global.openai.gpt-6-astra"
},
"openai.gpt-6-sol": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
@ -61883,6 +62074,9 @@
"/v1/responses"
]
},
"bedrock/gpt-luna-latest": {
"alias_of": "global.openai.gpt-6-luna"
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
@ -62160,6 +62354,9 @@
"supports_forced_tool_use": false,
"token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"}
},
"bedrock_mantle/claude-opus-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-opus-5-5"
},
"bedrock_mantle/anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 2.75e-06,
@ -62200,6 +62397,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock_mantle/claude-sonnet-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5"
},
"us.xai.grok-4.6": {
"supports_regex_lookaround": false,
"input_cost_per_token": 2.2e-06,
@ -63770,6 +63970,9 @@
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-flash-latest": {
"alias_of": "deepseek/deepseek-v4-flash"
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 6e-09,
@ -63878,6 +64081,9 @@
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"deepseek/deepseek-pro-latest": {
"alias_of": "deepseek/deepseek-v4-pro"
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 1.45e-07,
@ -65301,6 +65507,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/kimi-latest": {
"alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3"
},
"fireworks_ai/deepseek-v4-flash-0731": {
"cache_read_input_token_cost": 7e-09,
"cache_read_input_token_cost_priority": 8.75e-09,
@ -68974,6 +69183,9 @@
"output_cost_per_token": 4.7e-07,
"source": "https://api.together.ai/v1/models"
},
"together_ai/qwen-flash-latest": {
"alias_of": "together_ai/Qwen/Qwen3.8-Flash"
},
"together_ai/moonshotai/Kimi-K2.5-fp4": {
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.8e-06,
@ -69190,6 +69402,9 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"azure_ai/glm-latest": {
"alias_of": "azure_ai/FW-GLM-5.3"
},
"azure_ai/FW-GLM-5.3-Flash": {
"cache_read_input_token_cost": 3.8e-08,
"input_cost_per_token": 1.88e-07,
@ -69205,6 +69420,9 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"azure_ai/glm-flash-latest": {
"alias_of": "azure_ai/FW-GLM-5.3-Flash"
},
"azure_ai/FW-GPT-OSS-120B": {
"deprecation_date": "2027-07-01",
"cache_read_input_token_cost": 8.2e-08,
@ -71210,6 +71428,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/glm-latest": {
"alias_of": "baseten/zai-org/GLM-5.3"
},
"baseten/zai-org/GLM-5.3-Fast": {
"cache_read_input_token_cost": 2.1e-07,
"input_cost_per_token": 2.1e-06,
@ -71553,6 +71774,13 @@
"output_cost_per_token": 1.7e-06,
"source": "https://api.together.ai/v1/models"
},
"together_ai/essentialai/rnj-1-instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://www.together.ai/models/rnj-1-instruct"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_source": "https://docs.together.ai/docs/deprecations",
@ -74399,6 +74627,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/kimi-latest": {
"alias_of": "global.moonshotai.kimi-k3"
},
"us.moonshotai.kimi-k3": {
"supports_regex_lookaround": false,
"cache_creation_input_token_cost": 4.125e-06,
@ -74949,6 +75180,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/kimi-latest": {
"alias_of": "baseten/moonshotai/Kimi-K3"
},
"baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 6e-07,
@ -75031,6 +75265,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/glm-flash-latest": {
"alias_of": "baseten/zai-org/GLM-5.3-Flash"
},
"baseten/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.3e-07,
@ -75047,6 +75284,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"baseten/deepseek-flash-latest": {
"alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731"
},
"baseten/deepseek-ai/DeepSeek-V4-Pro": {
"deprecation_date": "2026-10-16",
"cache_read_input_token_cost": 1.45e-07,
@ -75080,6 +75320,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"baseten/deepseek-pro-latest": {
"alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"baseten/zai-org/GLM-5.2-Fast": {
"cache_read_input_token_cost": 2.1e-07,
"input_cost_per_token": 2.1e-06,
@ -76071,6 +76314,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01"
},
"azure_ai/claude-sonnet-latest": {
"alias_of": "azure_ai/claude-sonnet-5-5"
},
"bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -76235,6 +76481,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock/claude-sonnet-latest": {
"alias_of": "global.anthropic.claude-sonnet-5-5"
},
"jp.anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 2.75e-06,
@ -76494,6 +76743,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-sonnet-latest": {
"alias_of": "vertex_ai/claude-sonnet-5-5"
},
"vertex_ai/claude-sonnet-5-5@default": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
@ -76606,6 +76858,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/grok-latest": {
"alias_of": "global.xai.grok-4.7"
},
"us.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
@ -76778,6 +77033,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-sol-latest": {
"alias_of": "gpt-6.1-sol"
},
"global.openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -76824,6 +77082,9 @@
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"bedrock/gpt-sol-latest": {
"alias_of": "global.openai.gpt-6.1-sol"
},
"bedrock_mantle/openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
@ -76872,6 +77133,9 @@
"supports_xhigh_reasoning_effort": true,
"use_openai_responses_path": true
},
"bedrock_mantle/gpt-sol-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6.1-sol"
},
"us.openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
@ -76963,6 +77227,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"vertex_ai/grok-latest": {
"alias_of": "vertex_ai/xai/grok-4.7"
},
"azure_ai/kimi-k2-thinking": {
"deprecation_date": "2026-03-29",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
@ -77020,6 +77287,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"bedrock/glm-latest": {
"alias_of": "global.zai.glm-5.3"
},
"us.zai.glm-5.3": {
"cache_creation_input_token_cost": 2.31e-06,
"cache_creation_input_token_cost_batches": 1.155e-06,
@ -77243,6 +77513,9 @@
"supports_vision": true,
"supports_web_search": true
},
"azure_ai/grok-latest": {
"alias_of": "azure_ai/grok-4.7"
},
"in.moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_flex": 2.0625e-06,
@ -77329,6 +77602,9 @@
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08
},
"claude-haiku-latest": {
"alias_of": "claude-haiku-5-5"
},
"bedrock_mantle/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
@ -77375,6 +77651,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"bedrock_mantle/claude-haiku-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5"
},
"bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
@ -77606,6 +77885,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
},
"azure_ai/claude-haiku-latest": {
"alias_of": "azure_ai/claude-haiku-5-5"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
@ -77796,6 +78078,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock/claude-haiku-latest": {
"alias_of": "global.anthropic.claude-haiku-5-5"
},
"jp.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
@ -78003,6 +78288,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"vertex_ai/claude-haiku-latest": {
"alias_of": "vertex_ai/claude-haiku-5-5"
},
"vertex_ai/claude-haiku-5-5@default": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
@ -78907,6 +79195,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/grok-latest": {
"alias_of": "github_copilot/grok-4.7"
},
"together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
@ -80330,6 +80621,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-terra-latest": {
"alias_of": "gpt-6.1-terra"
},
"chatgpt/gpt-6.1-terra": {
"litellm_provider": "chatgpt",
"max_input_tokens": 922000,
@ -80388,6 +80682,9 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"bedrock/gpt-terra-latest": {
"alias_of": "global.openai.gpt-6.1-terra"
},
"us.openai.gpt-6.1-terra": {
"cache_creation_input_token_cost": 1.375e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.75e-06,
@ -80467,6 +80764,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-terra-latest": {
"alias_of": "github_copilot/gpt-6.1-terra"
},
"refuseless/glm-5.3-abliterated": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1e-06,
@ -80621,6 +80921,9 @@
"supports_xhigh_reasoning_effort": true,
"use_openai_responses_path": true
},
"bedrock_mantle/gpt-terra-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6.1-terra"
},
"clf_ai_gateway/glm-5.3": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
@ -82170,6 +82473,9 @@
"supports_vision": true,
"thinking_always_on": false
},
"databricks/claude-haiku-latest": {
"alias_of": "databricks/databricks-claude-haiku-5-5"
},
"databricks/databricks-claude-sonnet-5-5": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_creation_input_token_cost_above_1hr": 4.00001e-06,
@ -82199,6 +82505,9 @@
"supports_vision": false,
"thinking_always_on": true
},
"databricks/claude-sonnet-latest": {
"alias_of": "databricks/databricks-claude-sonnet-5-5"
},
"databricks/databricks-deepseek-v4-1-flash": {
"cache_creation_input_token_cost": 3.0002e-07,
"cache_read_input_token_cost": 3.003e-08,
@ -82431,6 +82740,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
},
"databricks/gpt-sol-latest": {
"alias_of": "databricks/databricks-gpt-6-1-sol"
},
"databricks/databricks-gpt-6-1-terra": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06,
@ -82471,6 +82783,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06
},
"databricks/gpt-terra-latest": {
"alias_of": "databricks/databricks-gpt-6-1-terra"
},
"databricks/databricks-gpt-6-astra": {
"cache_creation_input_token_cost": 1.249997e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
@ -82511,6 +82826,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05
},
"databricks/gpt-astra-latest": {
"alias_of": "databricks/databricks-gpt-6-astra"
},
"databricks/databricks-gpt-6-luna": {
"cache_creation_input_token_cost": 1.2502e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07,
@ -82551,6 +82869,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07
},
"databricks/gpt-luna-latest": {
"alias_of": "databricks/databricks-gpt-6-luna"
},
"databricks/databricks-gpt-6-sol": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06,
@ -82681,6 +83002,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/grok-latest": {
"alias_of": "databricks/databricks-grok-4-7"
},
"databricks/databricks-kimi-k2-7": {
"cache_creation_input_token_cost": 9.4997e-07,
"cache_read_input_token_cost": 1.8998e-07,
@ -86613,6 +86937,9 @@
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock_mantle/grok-latest": {
"alias_of": "bedrock_mantle/xai.grok-4.7"
},
"together_ai/ricursive-ai/Ripple-65B-Preview": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,

69
litellm/proxy/common_utils/encrypt_decrypt_utils.py
View file

@ -1,11 +1,14 @@
import base64
import ctypes
import os
import re
from collections.abc import Mapping
from typing import Final, Literal, TypeVar, cast, overload
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.proxy.auth.master_key_boot_check import SALT_KEY_ENV_VAR
# Versioned ciphertext marker for AES-256-GCM values.
# Format: "v2:gcm:" + base64url(nonce(12) || ciphertext || tag(16)).
@ -42,6 +45,72 @@ def get_salt_key() -> str | None:
_get_salt_key: Final = get_salt_key
_SALT_KEY_HEX: Final = re.compile(r"(?:sk-)?((?:[0-9a-fA-F]{2}){32,})")
_OSSL_PARAM_UTF8_STRING: Final = 4
_OSSL_PARAM_OCTET_STRING: Final = 5
_OSSL_PARAM_UNMODIFIED: Final = ctypes.c_size_t(-1).value
_DERIVED_KEY_BYTES: Final = 32
_KBKDF_PROPERTIES: Final = b"fips=yes"
class _OsslParam(ctypes.Structure):
_fields_ = (
("key", ctypes.c_char_p),
("data_type", ctypes.c_uint),
("data", ctypes.c_char_p),
("data_size", ctypes.c_size_t),
("return_size", ctypes.c_size_t),
)
def _param(name: bytes, data_type: int, data: bytes) -> _OsslParam:
return _OsslParam(name, data_type, data, len(data), _OSSL_PARAM_UNMODIFIED)
def _kbkdf_hmac_sha256(key: bytes, label: bytes, context: bytes) -> bytes:
import _hashlib
libcrypto: Final = ctypes.CDLL(_hashlib.__file__)
kdf_fetch: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_char_p)(
("EVP_KDF_fetch", libcrypto)
)
ctx_new: Final = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(("EVP_KDF_CTX_new", libcrypto))
derive: Final = ctypes.CFUNCTYPE(
ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_size_t, ctypes.POINTER(_OsslParam)
)(("EVP_KDF_derive", libcrypto))
ctx_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_CTX_free", libcrypto))
kdf_free: Final = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(("EVP_KDF_free", libcrypto))
params: Final = (_OsslParam * 6)(
_param(b"mac", _OSSL_PARAM_UTF8_STRING, b"HMAC"),
_param(b"digest", _OSSL_PARAM_UTF8_STRING, b"SHA256"),
_param(b"key", _OSSL_PARAM_OCTET_STRING, key),
_param(b"salt", _OSSL_PARAM_OCTET_STRING, label),
_param(b"info", _OSSL_PARAM_OCTET_STRING, context),
)
derived: Final = ctypes.create_string_buffer(_DERIVED_KEY_BYTES)
kdf: Final = kdf_fetch(None, b"KBKDF", _KBKDF_PROPERTIES)
ctx: Final = ctx_new(kdf)
try:
if not ctx or derive(ctx, derived, _DERIVED_KEY_BYTES, params) != 1:
raise ValueError("OpenSSL could not run KBKDF")
return derived.raw
finally:
ctx_free(ctx)
kdf_free(kdf)
def derive_key_from_salt_key(label: bytes, context: bytes) -> str:
"""Hex of 32 bytes derived from LITELLM_SALT_KEY under label and context by SP 800-108r1 KBKDF in counter mode
with HMAC-SHA256, run by EVP_KDF from the FIPS provider of the OpenSSL that hashlib uses. Raises ValueError when
LITELLM_SALT_KEY is not at least 32 bytes of hex, optionally prefixed with sk-, or when no FIPS provider runs it."""
salt_key: Final = _SALT_KEY_HEX.fullmatch(os.getenv(SALT_KEY_ENV_VAR, ""))
if salt_key is None:
raise ValueError(
f"With general_settings.fips_mode on, {SALT_KEY_ENV_VAR} must be at least 32 bytes of hex, such as "
"sk-$(openssl rand -hex 32)"
)
return _kbkdf_hmac_sha256(bytes.fromhex(salt_key.group(1)), label, context).hex()
def _get_encryption_algorithm() -> str:
"""

157
litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
View file

@ -6,15 +6,17 @@ to detect and block/mask sensitive content.
"""
import asyncio
import bisect
import contextvars
import itertools
import json
import os
import re
import time
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Mapping, Sequence
from collections.abc import AsyncGenerator, AsyncIterable, Callable, Coroutine, Iterable, Mapping, Sequence
from dataclasses import dataclass, replace
from datetime import datetime
from functools import partial
from re import Pattern
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, TypeVar, cast
@ -79,7 +81,7 @@ from .competitor_intent import (
AirlineCompetitorIntentChecker,
BaseCompetitorIntentChecker,
)
from .patterns import PATTERN_EXTRA_CONFIG, get_compiled_pattern
from .patterns import PATTERN_EXTRA_CONFIG, PATTERN_STREAMING_HOLDBACK_CHARS, get_compiled_pattern
MAX_KEYWORD_VALUE_GAP_WORDS: Final = 1
_DECODED_ARGUMENTS: Final = TypeAdapter(object)
@ -223,10 +225,70 @@ async def _stream_then_end_of_stream_chunk(
@dataclass(frozen=True, slots=True)
class _StreamedScanPlan:
context_chars: int
holdback_chars: int
exception_phrases: tuple[str, ...]
conditional_words: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class _RedactionSpan:
start: int
end: int
pattern_index: int
pattern_name: str
def _merge_redaction_spans(spans: Iterable[_RedactionSpan]) -> tuple[_RedactionSpan, ...]:
ordered: Final = sorted(spans, key=lambda span: (span.start, -span.end))
reach: Final = tuple(itertools.accumulate((span.end for span in ordered), max))
first_indices: Final = tuple(
index for index, span in enumerate(ordered) if index == 0 or span.start >= reach[index - 1]
)
last_indices: Final = (*(index - 1 for index in first_indices[1:]), len(ordered) - 1)
return tuple(replace(ordered[first], end=reach[last]) for first, last in zip(first_indices, last_indices))
@dataclass(frozen=True, slots=True)
class _RedactedText:
spans: tuple[_RedactionSpan, ...]
tags: tuple[str, ...]
tag_starts: tuple[int, ...]
def render(self, text: str) -> str:
previous_ends: Final = (0, *(span.end for span in self.spans))
kept_parts: Final = (
text[previous_end : span.start] + tag
for previous_end, span, tag in zip(previous_ends, self.spans, self.tags)
)
return "".join((*kept_parts, text[previous_ends[-1] :]))
def original_start(self, position: int) -> int:
index: Final = bisect.bisect_right(self.tag_starts, position) - 1
if index < 0:
return position
tag_end: Final = self.tag_starts[index] + len(self.tags[index])
if position < tag_end:
return self.spans[index].start
return position - tag_end + self.spans[index].end
def original_end(self, position: int) -> int:
index: Final = bisect.bisect_left(self.tag_starts, position) - 1
if index < 0:
return position
tag_end: Final = self.tag_starts[index] + len(self.tags[index])
if position <= tag_end:
return self.spans[index].end
return position - tag_end + self.spans[index].end
def _redacted_text(spans: tuple[_RedactionSpan, ...], redaction_format: str) -> _RedactedText:
tags: Final = tuple(redaction_format.format(pattern_name=span.pattern_name.upper()) for span in spans)
shifts: Final = itertools.accumulate(
(len(tag) - (span.end - span.start) for span, tag in zip(spans, tags)), initial=0
)
return _RedactedText(spans, tags, tuple(span.start + shift for span, shift in zip(spans, shifts)))
class CategoryFileData(TypedDict, total=False):
category_name: str
description: str
@ -987,21 +1049,6 @@ class ContentFilterGuardrail(CustomGuardrail):
merged.append((start, end))
return merged
def _mask_spans(self, text: str, spans: list[tuple[int, int]], redaction: str) -> str:
"""Apply masking for the provided spans using the given redaction tag."""
if not spans:
return text
result_parts: Final[list[str]] = []
previous_end = 0
for start, end in spans:
result_parts.append(text[previous_end:start])
result_parts.append(redaction)
previous_end = end
result_parts.append(text[previous_end:])
return "".join(result_parts)
def _convert_word_number_sequence(self, sequence: str) -> str | None:
"""Convert a spelled-out digit sequence (e.g., 'One-Two') into digits."""
@ -1392,10 +1439,8 @@ class ContentFilterGuardrail(CustomGuardrail):
self,
pattern_name: str,
action: ContentFilterAction,
text: str,
spans: list[tuple[int, int]],
detections: list[ContentFilterDetection] | None,
) -> str:
) -> None:
"""Handle regex pattern match detection and action."""
if detections is not None:
pattern_detection: Final[PatternDetection] = {
@ -1413,11 +1458,49 @@ class ContentFilterGuardrail(CustomGuardrail):
detail={"error": error_msg, "pattern": pattern_name},
)
elif action == ContentFilterAction.MASK:
redaction_tag: Final = self.pattern_redaction_format.format(pattern_name=pattern_name.upper())
text = self._mask_spans(text, spans, redaction_tag)
verbose_proxy_logger.info("Masked all %s matches in content", pattern_name)
return text
def _mask_step(
self,
text: str,
spans: tuple[_RedactionSpan, ...],
indexed_entry: tuple[int, CompiledPatternEntry],
) -> tuple[_RedactionSpan, ...]:
index, pattern_entry = indexed_entry
if pattern_entry["action"] != ContentFilterAction.MASK:
return spans
original_matches: Final = self._find_pattern_spans(text, pattern_entry)
redacted_matches: Final = self._redacted_text_matches(text, spans, pattern_entry) if spans else ()
return _merge_redaction_spans(
(
*spans,
*(
_RedactionSpan(start, end, index, pattern_entry["pattern_name"])
for start, end in (*original_matches, *redacted_matches)
),
)
)
def _redacted_text_matches(
self, text: str, spans: tuple[_RedactionSpan, ...], pattern_entry: CompiledPatternEntry
) -> tuple[tuple[int, int], ...]:
redacted: Final = _redacted_text(spans, self.pattern_redaction_format)
return tuple(
(redacted.original_start(start), redacted.original_end(end))
for start, end in self._find_pattern_spans(redacted.render(text), pattern_entry)
)
def _pattern_detected(
self,
text: str,
index: int,
pattern_entry: CompiledPatternEntry,
span_states: Sequence[tuple[_RedactionSpan, ...]],
) -> bool:
if pattern_entry["action"] == ContentFilterAction.MASK:
return any(span.pattern_index == index for span in span_states[-1])
text_after_earlier_masks: Final = _redacted_text(span_states[index], self.pattern_redaction_format).render(text)
return bool(self._find_pattern_spans(text_after_earlier_masks, pattern_entry))
def _handle_blocked_word_match(
self,
@ -1506,13 +1589,14 @@ class ContentFilterGuardrail(CustomGuardrail):
keyword, category_name, severity, action = category_keyword_match
text = self._handle_category_keyword_match(keyword, category_name, severity, action, text, detections)
# Check regex patterns - process ALL patterns, not just first match
for pattern_entry in self.compiled_patterns:
spans = self._find_pattern_spans(text, pattern_entry)
if spans:
pattern_name = pattern_entry["pattern_name"]
action = pattern_entry["action"]
text = self._handle_pattern_match(pattern_name, action, text, spans, detections)
no_spans: Final[tuple[_RedactionSpan, ...]] = ()
span_states: Final = tuple(
itertools.accumulate(enumerate(self.compiled_patterns), partial(self._mask_step, text), initial=no_spans)
)
for index, pattern_entry in enumerate(self.compiled_patterns):
if self._pattern_detected(text, index, pattern_entry, span_states):
self._handle_pattern_match(pattern_entry["pattern_name"], pattern_entry["action"], detections)
text = _redacted_text(span_states[-1], self.pattern_redaction_format).render(text)
# Check blocked words - iterate through ALL blocked words
text_lower = text.lower()
@ -2071,7 +2155,9 @@ class ContentFilterGuardrail(CustomGuardrail):
def _streamed_scan_plan(self) -> _StreamedScanPlan:
"""
Per-stream inputs for buffer trimming: the retained tail length (the default
context, widened to the longest configured keyword), the category exception
context, widened to the longest configured keyword), the unreleased tail length
(widened to the longest holdback a configured prebuilt pattern declares, since
provider keys run longer than the default), the category exception
phrases, which suppress matches anywhere in the scanned text, and the conditional
category words, which only match when paired inside one sentence.
"""
@ -2081,6 +2167,13 @@ class ContentFilterGuardrail(CustomGuardrail):
)
return _StreamedScanPlan(
context_chars=max(CONTENT_FILTER_STREAMING_SCAN_CONTEXT_CHARS, longest_keyword),
holdback_chars=max(
(
PATTERN_STREAMING_HOLDBACK_CHARS.get(entry["pattern_name"], CONTENT_FILTER_STREAMING_HOLDBACK_CHARS)
for entry in self.compiled_patterns
),
default=CONTENT_FILTER_STREAMING_HOLDBACK_CHARS,
),
exception_phrases=tuple(
phrase for category in self.loaded_categories.values() for phrase in category.exceptions
),
@ -2221,7 +2314,7 @@ class ContentFilterGuardrail(CustomGuardrail):
safe_to_yield_len = max(
previous_state.yielded_masked_text_len,
len(masked_text) - (0 if is_final else CONTENT_FILTER_STREAMING_HOLDBACK_CHARS),
len(masked_text) - (0 if is_final else plan.holdback_chars),
)
released_text = masked_text[previous_state.yielded_masked_text_len : safe_to_yield_len]
if isinstance(choice, TextChoices):

24
litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.json
View file

@ -98,6 +98,28 @@
"category": "Credential Patterns",
"description": "Detects generic API key patterns"
},
{
"name": "openai_api_key",
"display_name": "OpenAI API Key",
"pattern": "sk-(?:(?:proj|svcacct|admin|None)-[A-Za-z0-9_\\-]{20,200}T3BlbkFJ(?:(?!sk-(?:proj|svcacct|admin|None)-)[A-Za-z0-9_\\-]){20,200}|[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20}(?![A-Za-z0-9_]))",
"streaming_holdback_chars": 419,
"category": "Credential Patterns",
"description": "Detects OpenAI API keys (sk-proj-..., sk-svcacct-..., sk-admin-..., sk-None-... and legacy sk-... keys, all carrying the T3BlbkFJ marker)"
},
{
"name": "anthropic_api_key",
"display_name": "Anthropic API Key",
"pattern": "(?:(?<![A-Za-z0-9])|(?<=\\\\[nrt])|(?<=\\\\u[0-9A-F]{4})|(?<=%[0-9A-F]{2})|(?<=%25[0-9A-F]{2}))sk-ant-[A-Za-z0-9_\\-]{20,}",
"category": "Credential Patterns",
"description": "Detects Anthropic API keys (sk-ant-... format)"
},
{
"name": "gemini_api_key",
"display_name": "Gemini / Google API Key",
"pattern": "(?:(?<![A-Za-z0-9])|(?<=\\\\[nrt])|(?<=\\\\u[0-9A-F]{4})|(?<=%[0-9A-F]{2})|(?<=%25[0-9A-F]{2}))(?:(?-i:AIza)[A-Za-z0-9_\\-]{33,36}|(?-i:AQ\\.)[A-Za-z0-9_\\-]{45,55})(?![A-Za-z0-9_])",
"category": "Credential Patterns",
"description": "Detects Google / Gemini API keys (legacy AIza... and new AQ.... authorization key formats)"
},
{
"name": "ipv4",
"display_name": "IP Address (IPv4)",
@ -626,4 +648,4 @@
"allow_word_numbers": false
}
]
}
}

13
litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/patterns.py
View file

@ -11,10 +11,11 @@ import re
from collections.abc import Iterator, Sequence
from enum import Enum
from re import Pattern
from types import MappingProxyType
from typing import Final
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import ReadOnly, TypedDict
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.proxy.guardrails.content_filter_data import DATA_ROOTS, category_dirs
@ -26,6 +27,7 @@ class _PatternEntry(TypedDict):
pattern: ReadOnly[str]
category: ReadOnly[str]
description: ReadOnly[str]
streaming_holdback_chars: ReadOnly[NotRequired[int]]
@with_config(ConfigDict(extra="allow", strict=True, hide_input_in_errors=True))
@ -84,6 +86,7 @@ KNOWN_PATTERN_KEYS: Final = {
"category",
"action",
"description",
"streaming_holdback_chars",
}
PATTERN_EXTRA_CONFIG: Final[dict[str, _PatternExtraConfig]] = {}
@ -132,6 +135,14 @@ for pattern_data in _PATTERNS_DATA["patterns"]:
PATTERN_CATEGORIES[category] = []
PATTERN_CATEGORIES[category].append(pattern_data["name"])
PATTERN_STREAMING_HOLDBACK_CHARS: Final = MappingProxyType(
{
pattern_data["name"]: pattern_data["streaming_holdback_chars"]
for pattern_data in _PATTERNS_DATA["patterns"]
if "streaming_holdback_chars" in pattern_data
}
)
# Build display names mapping from JSON
PATTERN_DISPLAY_NAMES: Final[dict[str, str]] = {

10
litellm/proxy/mcp_server/outbound_credentials/bridge_credentials.py
View file

@ -16,6 +16,8 @@ from typing import Final, Literal, TypeAlias
from pydantic import ConfigDict, SecretStr
from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key
from litellm.proxy.common_utils.fips import fips_mode_active
from litellm.proxy.mcp_server.outbound_credentials.envelope import (
EnvelopeIdentity,
EnvelopeKeys,
@ -61,7 +63,15 @@ def envelope_keys_from_master_key(master_key: str) -> EnvelopeKeys:
fixed for a process), so the KDF runs once per key and adds nothing to the per-request
admission path. The derivation is deterministic; rotating ``master_key`` invalidates
every outstanding envelope, which is the intended behavior for a signing-key change.
When general_settings.fips_mode is on, both keys come from :func:`derive_key_from_salt_key` with the master key as
context instead.
"""
if fips_mode_active():
return EnvelopeKeys(
signing_key=SecretStr(derive_key_from_salt_key(_SIGNING_KEY_DOMAIN, master_key.encode())),
encryption_key=SecretStr(derive_key_from_salt_key(_ENCRYPTION_KEY_DOMAIN, master_key.encode())),
)
signing: Final = hashlib.scrypt(
master_key.encode(),
salt=_SIGNING_KEY_DOMAIN,

9
litellm/proxy/mcp_server/outbound_credentials/session_credentials.py
View file

@ -22,6 +22,8 @@ from typing import Final, Literal, TypeAlias
from pydantic import ConfigDict, Field, SecretStr, ValidationError
from litellm.proxy.common_utils.encrypt_decrypt_utils import derive_key_from_salt_key
from litellm.proxy.common_utils.fips import fips_mode_active
from litellm.proxy.mcp_server.outbound_credentials.session_token import (
AsymmetricSessionKeys,
OpenedSessionToken,
@ -59,7 +61,14 @@ def session_keys_from_master_key(master_key: str) -> SessionKeys:
into the other. The result is cached (the master key is fixed for a process); rotating
``master_key`` invalidates every outstanding session, which is the intended behavior
for a signing-key change.
When general_settings.fips_mode is on, the key comes from :func:`derive_key_from_salt_key` with the master key as
context instead.
"""
if fips_mode_active():
return SessionKeys(
signing_key=SecretStr(derive_key_from_salt_key(_SESSION_SIGNING_KEY_DOMAIN, master_key.encode()))
)
signing: Final = hashlib.scrypt(
master_key.encode(),
salt=_SESSION_SIGNING_KEY_DOMAIN,

41
litellm/responses/bridge_check.py
View file

@ -4,7 +4,10 @@ from urllib.parse import urlsplit
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
resolve_model_and_provider_for_metadata,
)
from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
from litellm.llms.azure_ai.common_utils import (
azure_ai_supports_native_responses,
@ -291,3 +294,39 @@ def responses_api_bridge_check(
model = model.replace("responses/", "")
return model_info, model
def chat_completion_ends_in_responses_api(
model: str | None,
custom_llm_provider: str | None,
deployment_model_info: Mapping[str, object] | None = None,
*,
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
reasoning_summary: object | None = None,
api_base: str | None = None,
) -> bool:
"""Whether chat completion for this model is served by a Responses API.
Encrypted reasoning can be replayed only there. A normal chat provider cannot
verify the block, so the messages adapter strips it. ``chatgpt``, a ``responses/``
model, a deployment pinned to ``mode: responses``, and a request whose tools or
reasoning effort the bridge sends to Responses keep it.
"""
if not model:
return False
try:
resolved_model, provider = resolve_model_and_provider_for_metadata(model, custom_llm_provider)
except Exception as error: # noqa: BLE001 # unknown model names must not fail message translation
verbose_logger.debug("responses replay check failed for %s: %s", model, error)
return False
model_info, _resolved = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
tools=tools,
reasoning_effort=reasoning_effort,
reasoning_summary=reasoning_summary,
api_base=api_base,
deployment_model_info=deployment_model_info,
)
return model_info.get("mode") == "responses"

11
litellm/responses/streaming_iterator.py
View file

@ -486,9 +486,18 @@ class BaseResponsesAPIStreamingIterator:
if _is_json_object(parsed_chunk):
if self.responses_api_provider_config is None:
raise ValueError("responses_api_provider_config is required to process live streaming chunks")
typed_chunk: Final = (
parsed_chunk
if parsed_chunk.get("type") is not None or "error" not in parsed_chunk
else {
"sequence_number": SEQUENCE_NUMBER_STAMPED_ON_EMISSION,
**parsed_chunk,
"type": ResponsesAPIStreamEvents.ERROR.value,
}
)
openai_responses_api_chunk: Final = self.responses_api_provider_config.transform_streaming_response(
model=self.model,
parsed_chunk=parsed_chunk,
parsed_chunk=typed_chunk,
logging_obj=self.logging_obj,
)

1
litellm/rust_bridge/host/model_capabilities.py
View file

@ -27,6 +27,7 @@ def anthropic_model_capabilities(model: str, custom_llm_provider: str | None) ->
"supports_reasoning": supports("supports_reasoning"),
"supports_adaptive_thinking": supports("supports_adaptive_thinking"),
"thinking_always_on": supports("thinking_always_on"),
"thinking_on_by_default": AnthropicModelInfo.thinks_by_default(model, provider),
"supports_legacy_thinking": supports("supports_legacy_thinking"),
"supports_output_config": supports("supports_output_config"),
"supports_sampling_params": AnthropicModelInfo._supports_sampling_params(resolved_model), # pyright: ignore[reportPrivateUsage] # same gate the handler applies

58
litellm/utils.py
View file

@ -53,7 +53,7 @@ import litellm.litellm_core_utils
# audio_utils.utils is lazy-loaded - only imported when needed for transcription calls
import litellm.litellm_core_utils.json_validation_rule
from litellm._internal_context import is_internal_call
from litellm._internal_context import is_internal_call, is_resent_failure
from litellm._lazy_imports import (
_get_modified_max_tokens,
get_default_encoding,
@ -2324,12 +2324,16 @@ def client(original_function):
retry_policy=kwargs.get("retry_policy"),
)
kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops
litellm.num_retries = None # set retries to None to prevent infinite loops
if not is_resent_failure(e):
litellm.num_retries = None # set retries to None to prevent infinite loops
context_window_fallback_dict: Final = kwargs.get("context_window_fallback_dict", {})
is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e)
num_retries
and not is_completion_litellm_router_call
and not is_agentic_followup_failure(e)
and not is_resent_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2359,11 +2363,15 @@ def client(original_function):
retry_policy=kwargs.get("retry_policy"),
)
kwargs["retry_policy"] = reset_retry_policy() # prevent infinite loops
litellm.num_retries = None # set retries to None to prevent infinite loops
if not is_resent_failure(e):
litellm.num_retries = None # set retries to None to prevent infinite loops
is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False)
if (
num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e)
num_retries
and not is_responses_litellm_router_call
and not is_agentic_followup_failure(e)
and not is_resent_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
if (
isinstance(e, openai.APIError)
@ -2376,7 +2384,7 @@ def client(original_function):
end_time = datetime.datetime.now()
# LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated
if logging_obj:
if logging_obj and not is_resent_failure(e):
logging_obj.failure_handler(
e, traceback_exception, start_time, end_time
) # DO NOT MAKE THREADED - router retry fallback relies on this!
@ -2641,7 +2649,7 @@ def client(original_function):
# Reuse the timestamp taken right when the deployment call itself failed, before
# the failure hook ran, so a slow callback doesn't inflate the reported duration.
end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now() # noqa: DTZ005 # matches the naive datetimes this whole function already times start_time/end_time with
if logging_obj and not _is_litellm_internal_call:
if logging_obj and not _is_litellm_internal_call and not is_resent_failure(e):
unbind_budget_reservation_from_callbacks(logging_obj.litellm_params)
try:
logging_obj.failure_handler(
@ -2666,6 +2674,7 @@ def client(original_function):
and not is_acompletion_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
and not is_resent_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops
@ -2676,8 +2685,9 @@ def client(original_function):
elif isinstance(e, openai.APIError): # generic api error
kwargs["retry_strategy"] = "constant_retry"
result = await litellm.acompletion_with_retries(*args, **kwargs)
except Exception:
pass
except Exception as retry_error:
if is_resent_failure(retry_error):
raise
else:
return result
elif (
@ -2700,6 +2710,7 @@ def client(original_function):
and not is_aresponses_litellm_router_call
and not isinstance(e, ImportError)
and not is_agentic_followup_failure(e)
and not is_resent_failure(e)
): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying
try:
litellm.num_retries = None # set retries to None to prevent infinite loops
@ -2710,8 +2721,9 @@ def client(original_function):
elif isinstance(e, openai.APIError): # generic api error
kwargs["retry_strategy"] = "constant_retry"
result = await litellm.aresponses_with_retries(*args, **kwargs)
except Exception:
pass
except Exception as retry_error:
if is_resent_failure(retry_error):
raise
else:
return result
@ -5496,6 +5508,10 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl
_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking")
_DROP_UNREAD_OUTPUT_CONFIG_WARNING: Final = (
"Dropping `output_config` for model=%s (drop_params=True) because the %s chat transformation does not read it."
)
def _turns_reasoning_off(value: object) -> bool:
match value:
@ -6167,6 +6183,13 @@ def get_optional_params(
drop_params=True,
custom_llm_provider=custom_llm_provider or None,
)
if (
"output_config" in passed_params
and not _declared_reasoning_efforts(model_info)
and not (isinstance(provider_config, litellm.AnthropicConfig) or "output_config" in supported_params)
):
litellm.verbose_logger.warning(_DROP_UNREAD_OUTPUT_CONFIG_WARNING, model, custom_llm_provider)
passed_params.pop("output_config")
# if user passed in non-default kwargs for specific providers/models, pass them along
optional_params = add_provider_specific_params_to_optional_params(
optional_params=optional_params,
@ -12048,15 +12071,6 @@ def should_run_mock_completion(
def __getattr__(name: str) -> Any:
"""Lazy import handler for utils module with cached registry for improved performance."""
# Use cached registry from _lazy_imports instead of importing tuples every time
from litellm._lazy_imports import get_lazy_import_registry
from litellm._lazy_imports import resolve_lazy_attribute
registry: Final = get_lazy_import_registry()
# Check if name is in registry and call the cached handler function
if name in registry:
handler_func: Final = registry[name]
return handler_func(name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
return resolve_lazy_attribute(__name__, name)

339
model_prices_and_context_window.json
View file

@ -1969,6 +1969,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://aws.amazon.com/bedrock/pricing/"
},
"bedrock/claude-fable-latest": {
"alias_of": "global.anthropic.claude-fable-5-1"
},
"us.anthropic.claude-fable-5": {
"cache_creation_input_token_cost": 1.375e-05,
"cache_creation_input_token_cost_above_1hr": 2.2e-05,
@ -2305,6 +2308,9 @@
"region": "us-east-1"
}
},
"bedrock/claude-opus-latest": {
"alias_of": "global.anthropic.claude-opus-5-5"
},
"us.anthropic.claude-opus-5": {
"bedrock_converse_supports_strict_tools": false,
"supports_adaptive_thinking": true,
@ -4063,6 +4069,9 @@
"deprecation_date": "2027-12-05",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement"
},
"azure_ai/claude-fable-latest": {
"alias_of": "azure_ai/claude-fable-5-1"
},
"azure_ai/claude-mythos-5-1": {
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
@ -4165,6 +4174,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01"
},
"azure_ai/claude-opus-latest": {
"alias_of": "azure_ai/claude-opus-5-5"
},
"azure_ai/claude-opus-4-8": {
"deprecation_date": "2027-09-01",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -4382,6 +4394,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-astra-latest": {
"alias_of": "azure_ai/gpt-6-astra"
},
"azure_ai/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -4430,6 +4445,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-luna-latest": {
"alias_of": "azure_ai/gpt-6-luna"
},
"azure_ai/gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -4526,6 +4544,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-sol-latest": {
"alias_of": "azure_ai/gpt-6.1-sol"
},
"azure_ai/gpt-6.1-terra": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-06,
@ -4574,6 +4595,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure_ai/gpt-terra-latest": {
"alias_of": "azure_ai/gpt-6.1-terra"
},
"azure_ai/gpt-5.5": {
"deprecation_date": "2027-10-26",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -5036,6 +5060,9 @@
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"azure_ai/gpt-mini-latest": {
"alias_of": "azure_ai/gpt-5.4-mini"
},
"azure_ai/gpt-5.4-mini-2026-03-17": {
"deprecation_date": "2027-09-21",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -9080,6 +9107,9 @@
"cache_read_input_token_cost_flex": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-06
},
"azure/gpt-astra-latest": {
"alias_of": "azure/gpt-6-astra"
},
"azure/gpt-6-astra-2026-09-03": {
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
@ -9184,6 +9214,9 @@
"output_cost_per_token_above_272k_tokens_flex": 3.75e-07,
"output_cost_per_token_flex": 2.5e-07
},
"azure/gpt-luna-latest": {
"alias_of": "azure/gpt-6-luna"
},
"azure/gpt-6-luna-2026-09-22": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -9403,6 +9436,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-terra-latest": {
"alias_of": "azure/gpt-6.1-terra"
},
"azure/gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -9455,6 +9491,9 @@
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06
},
"azure/gpt-sol-latest": {
"alias_of": "azure/gpt-6.1-sol"
},
"azure/gpt-6.1-sol-2026-09-29": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -11146,6 +11185,9 @@
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
"supports_xhigh_reasoning_effort": true
},
"azure/gpt-mini-latest": {
"alias_of": "azure/gpt-5.4-mini"
},
"azure/gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_flex": 3.75e-08,
@ -13212,6 +13254,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/kimi-latest": {
"alias_of": "azure_ai/FW-Kimi-K3"
},
"azure_ai/FW-MiniMax-M2.5": {
"deprecation_date": "2027-07-01",
"cache_read_input_token_cost": 3.3e-08,
@ -13251,6 +13296,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"azure_ai/minimax-latest": {
"alias_of": "azure_ai/FW-MiniMax-M3"
},
"azure_ai/FW-Nemotron-Lightning-3.5-30B-A3B": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 6e-08,
@ -13656,6 +13704,9 @@
"cache_read_input_token_cost": 1.45e-07,
"supports_prompt_caching": true
},
"azure_ai/deepseek-pro-latest": {
"alias_of": "azure_ai/deepseek-v4-pro"
},
"azure_ai/deepseek-v4-flash": {
"deprecation_date": "2028-02-20",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/concepts/model-lifecycle-retirement",
@ -13674,6 +13725,9 @@
"cache_read_input_token_cost": 2.8e-08,
"supports_prompt_caching": true
},
"azure_ai/deepseek-flash-latest": {
"alias_of": "azure_ai/deepseek-v4-flash"
},
"azure_ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 1.4e-08,
"deprecation_date": "2026-12-03",
@ -16623,6 +16677,9 @@
"source": "https://platform.claude.com/docs/en/about-claude/pricing",
"supports_web_search": true
},
"claude-sonnet-latest": {
"alias_of": "claude-sonnet-5-5"
},
"claude-sonnet-4-6": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17024,6 +17081,9 @@
"supports_native_structured_output": true,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-fable-latest": {
"alias_of": "claude-fable-5-1"
},
"claude-opus-5-5": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17069,6 +17129,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/about-claude/pricing"
},
"claude-opus-latest": {
"alias_of": "claude-opus-5-5"
},
"claude-opus-5": {
"supports_web_search": true,
"supports_anthropic_compaction": true,
@ -17539,6 +17602,8 @@
"supports_reasoning": true
},
"codestral/codestral-2405": {
"deprecation_date": "2025-06-16",
"deprecation_source": "https://docs.mistral.ai/getting-started/models/",
"input_cost_per_token": 0.0,
"litellm_provider": "codestral",
"max_input_tokens": 32000,
@ -18866,6 +18931,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"dashscope/qwen-max-latest": {
"alias_of": "dashscope/qwen3.8-max"
},
"dashscope/qwen3.8-flash": {
"cache_creation_input_token_cost": 2e-07,
"cache_read_input_token_cost": 1.6e-08,
@ -18885,6 +18953,9 @@
"supports_vision": true,
"supports_web_search": true
},
"dashscope/qwen-flash-latest": {
"alias_of": "dashscope/qwen3.8-flash"
},
"dashscope/qwen3.8-omni-flash": {
"cache_read_input_token_cost": 1.6e-08,
"input_cost_per_token": 1.5e-07,
@ -21765,6 +21836,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"databricks/claude-fable-latest": {
"alias_of": "databricks/databricks-claude-fable-5-1"
},
"databricks/databricks-claude-haiku-4-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_creation_input_token_cost_above_1hr": 1.99997e-06,
@ -22005,6 +22079,9 @@
"supports_xhigh_reasoning_effort": true,
"thinking_always_on": true
},
"databricks/claude-opus-latest": {
"alias_of": "databricks/databricks-claude-opus-5-5"
},
"databricks/databricks-claude-sonnet-4": {
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
@ -22722,6 +22799,9 @@
"cache_read_input_token_cost_priority": 4.55e-07,
"cache_creation_input_token_cost_priority": 2.45e-06
},
"databricks/glm-latest": {
"alias_of": "databricks/databricks-glm-5-3"
},
"databricks/databricks-glm-5-3-flash": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 3.003e-08,
@ -22755,6 +22835,9 @@
"cache_read_input_token_cost_priority": 5.25e-08,
"cache_creation_input_token_cost_priority": 2.625e-07
},
"databricks/glm-flash-latest": {
"alias_of": "databricks/databricks-glm-5-3-flash"
},
"databricks/databricks-gpt-5": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_read_input_token_cost": 1.2502e-07,
@ -22917,6 +23000,9 @@
"cache_read_input_token_cost_priority": 1.5001e-07,
"cache_creation_input_token_cost_priority": 1.50003e-06
},
"databricks/gpt-mini-latest": {
"alias_of": "databricks/databricks-gpt-5-4-mini"
},
"databricks/databricks-gpt-5-4-nano": {
"cache_creation_input_token_cost": 1.9999e-07,
"cache_read_input_token_cost": 2.002e-08,
@ -23313,6 +23399,9 @@
"cache_read_input_token_cost_priority": 5.25e-07,
"cache_creation_input_token_cost_priority": 5.25e-06
},
"databricks/kimi-latest": {
"alias_of": "databricks/databricks-kimi-k3"
},
"databricks/system.ai.kimi-k3": {
"litellm_provider": "databricks",
"mode": "chat",
@ -28841,6 +28930,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/minimax-latest": {
"alias_of": "fireworks_ai/accounts/fireworks/models/minimax-m3"
},
"fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
"input_cost_per_token": 1.2e-06,
"litellm_provider": "fireworks_ai",
@ -29341,7 +29433,7 @@
"source": "https://api.friendli.ai/serverless/v1/models"
},
"ft:babbage-002": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 1.6e-06,
@ -29355,7 +29447,7 @@
"source": "https://developers.openai.com/api/docs/pricing"
},
"ft:davinci-002": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 1.2e-05,
@ -29414,7 +29506,7 @@
"supports_tool_choice": true
},
"ft:gpt-3.5-turbo-1106": {
"deprecation_date": "2026-10-23",
"deprecation_date": "2026-09-28",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
"successor_model": "gpt-5.6-terra",
"input_cost_per_token": 3e-06,
@ -33787,6 +33879,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-fable-latest": {
"alias_of": "github_copilot/claude-fable-5.1"
},
"github_copilot/claude-haiku-4.5": {
"cache_creation_input_token_cost": 1.25e-06,
"cache_read_input_token_cost": 1e-07,
@ -33830,6 +33925,9 @@
"supports_vision": true,
"thinking_always_on": false
},
"github_copilot/claude-haiku-latest": {
"alias_of": "github_copilot/claude-haiku-5.5"
},
"github_copilot/claude-opus-4.8": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_read_input_token_cost": 5e-07,
@ -33918,6 +34016,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-opus-latest": {
"alias_of": "github_copilot/claude-opus-5.5"
},
"github_copilot/claude-sonnet-5": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_read_input_token_cost": 2e-07,
@ -33961,6 +34062,9 @@
"supports_vision": true,
"thinking_always_on": true
},
"github_copilot/claude-sonnet-latest": {
"alias_of": "github_copilot/claude-sonnet-5.5"
},
"github_copilot/gemini-3.7-flash": {
"cache_read_input_token_cost": 7.5e-08,
"deprecation_date": "2026-10-19",
@ -34306,6 +34410,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-astra-latest": {
"alias_of": "github_copilot/gpt-6-astra"
},
"github_copilot/gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
@ -34329,6 +34436,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-luna-latest": {
"alias_of": "github_copilot/gpt-6-luna"
},
"github_copilot/gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -34375,6 +34485,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-sol-latest": {
"alias_of": "github_copilot/gpt-6.1-sol"
},
"github_copilot/kimi-k3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
@ -34392,6 +34505,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/kimi-latest": {
"alias_of": "github_copilot/kimi-k3"
},
"github_copilot/mai-code-1.1-flash": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,
@ -37685,6 +37801,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-astra-latest": {
"alias_of": "gpt-6-astra"
},
"gpt-6-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_12hr": 5e-06,
@ -37882,6 +38001,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-luna-latest": {
"alias_of": "gpt-6-luna"
},
"gpt-5.6": {
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_12hr": 1e-05,
@ -39065,6 +39187,9 @@
"supports_xhigh_reasoning_effort": true,
"supports_minimal_reasoning_effort": false
},
"gpt-mini-latest": {
"alias_of": "gpt-5.4-mini"
},
"gpt-5.4-mini-2026-03-17": {
"cache_read_input_token_cost": 7.5e-08,
"cache_read_input_token_cost_batches": 3.75e-08,
@ -43023,6 +43148,9 @@
"max_input_tokens": 1000000,
"max_output_tokens": 128000
},
"minimax/minimax-latest": {
"alias_of": "minimax/MiniMax-M3"
},
"minimax/MiniMax-H3": {
"litellm_provider": "minimax",
"mode": "video_generation",
@ -44487,6 +44615,28 @@
"supports_response_schema": true,
"supports_vision": true
},
"moonshot/kimi-k3.1-pro-highspeed": {
"cache_creation_input_token_cost": 0.000004,
"cache_creation_input_token_cost_above_1hr": 0.000008,
"cache_read_input_token_cost": 2e-7,
"input_cost_per_token": 0.000004,
"litellm_provider": "moonshot",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 0.00002,
"reasoning_effort_levels": [
"low",
"high",
"max"
],
"source": "https://platform.kimi.ai/docs/pricing/chat",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true
},
"moonshot/kimi-k3": {
"cache_creation_input_token_cost": 3e-06,
"cache_creation_input_token_cost_above_1hr": 6e-06,
@ -44511,6 +44661,9 @@
"supports_video_input": true,
"supports_vision": true
},
"moonshot/kimi-latest": {
"alias_of": "moonshot/kimi-k3"
},
"moonshot/moonshot-v1-128k": {
"deprecation_date": "2026-08-31",
"input_cost_per_token": 2e-06,
@ -49673,6 +49826,8 @@
"mode": "search"
},
"text-completion-codestral/codestral-2405": {
"deprecation_date": "2025-06-16",
"deprecation_source": "https://docs.mistral.ai/getting-started/models/",
"input_cost_per_token": 0.0,
"litellm_provider": "text-completion-codestral",
"max_input_tokens": 32000,
@ -50021,13 +50176,13 @@
"supports_reasoning": true
},
"together_ai/Qwen/Qwen3.7-Max": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.5e-06,
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 4.5e-06,
"output_cost_per_token": 7.5e-06,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
@ -50110,6 +50265,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/deepseek-pro-latest": {
"alias_of": "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"together_ai/meta-models/Muse-Glimmer-30B": {
"cache_read_input_token_cost": 4e-08,
"input_cost_per_token": 3.5e-07,
@ -50156,6 +50314,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/kimi-latest": {
"alias_of": "together_ai/moonshotai/Kimi-K3"
},
"together_ai/thinkingmachines/Inkling": {
"cache_read_input_token_cost": 1.7e-07,
"deprecation_date": "2026-11-06",
@ -50206,6 +50367,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"together_ai/glm-latest": {
"alias_of": "together_ai/zai-org/GLM-5.3"
},
"together_ai/zai-org/GLM-5.3-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 1.5e-07,
@ -50223,6 +50387,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/glm-flash-latest": {
"alias_of": "together_ai/zai-org/GLM-5.3-Flash"
},
"tts-1": {
"deprecation_date": "2027-01-06",
"deprecation_source": "https://developers.openai.com/api/docs/deprecations",
@ -52993,6 +53160,9 @@
"output_cost_per_token_batches": 2.5e-05,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-fable-latest": {
"alias_of": "vertex_ai/claude-fable-5-1"
},
"vertex_ai/claude-fable-5@default": {
"deprecation_date": "2027-06-08",
"regional_endpoint_uplift_multiplier": 1.1,
@ -53191,6 +53361,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-opus-latest": {
"alias_of": "vertex_ai/claude-opus-5-5"
},
"vertex_ai/claude-opus-5-5@default": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
@ -55739,6 +55912,9 @@
"supports_vision": true,
"supports_web_search": true
},
"xai/grok-latest": {
"alias_of": "xai/grok-4.7"
},
"zai.glm-4.7": {
"deprecation_date": "2027-03-30",
"deprecation_source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle-legacy.html",
@ -55840,6 +56016,9 @@
"supports_response_schema": true,
"supports_tool_choice": true
},
"zai/glm-latest": {
"alias_of": "zai/glm-5.3"
},
"zai/glm-5.3-flash": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 3e-08,
@ -55857,6 +56036,9 @@
"source": "https://docs.z.ai/guides/overview/pricing",
"supports_vision": true
},
"zai/glm-flash-latest": {
"alias_of": "zai/glm-5.3-flash"
},
"zai/glm-5v-turbo": {
"cache_read_input_token_cost": 2.4e-07,
"input_cost_per_token": 1.2e-06,
@ -61493,6 +61675,9 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html",
"supports_web_search": true
},
"bedrock_mantle/gpt-astra-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6-astra"
},
"bedrock_mantle/openai.gpt-6-sol": {
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
@ -61573,6 +61758,9 @@
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
"supports_web_search": true
},
"bedrock_mantle/gpt-luna-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6-luna"
},
"us.openai.gpt-6-astra": {
"supports_bedrock_runtime_chat_completions_response_format": true,
"input_cost_per_token": 1.1e-05,
@ -61741,6 +61929,9 @@
"/v1/responses"
]
},
"bedrock/gpt-astra-latest": {
"alias_of": "global.openai.gpt-6-astra"
},
"openai.gpt-6-sol": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
@ -61883,6 +62074,9 @@
"/v1/responses"
]
},
"bedrock/gpt-luna-latest": {
"alias_of": "global.openai.gpt-6-luna"
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
@ -62160,6 +62354,9 @@
"supports_forced_tool_use": false,
"token_count_model": {"model":"anthropic.claude-opus-5-5","litellm_provider":"bedrock_mantle","region":"us-east-1"}
},
"bedrock_mantle/claude-opus-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-opus-5-5"
},
"bedrock_mantle/anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 2.75e-06,
@ -62200,6 +62397,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock_mantle/claude-sonnet-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-sonnet-5-5"
},
"us.xai.grok-4.6": {
"supports_regex_lookaround": false,
"input_cost_per_token": 2.2e-06,
@ -63770,6 +63970,9 @@
"token_count_framing": 23,
"token_count_tool_framing": 12
},
"deepseek/deepseek-flash-latest": {
"alias_of": "deepseek/deepseek-v4-flash"
},
"deepseek/deepseek-v4-flash-vision-exp": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 6e-09,
@ -63878,6 +64081,9 @@
"token_count_framing": 76,
"token_count_tool_framing": 75
},
"deepseek/deepseek-pro-latest": {
"alias_of": "deepseek/deepseek-v4-pro"
},
"tencent/deepseek-v4-pro": {
"cache_creation_input_token_cost": 0.0,
"cache_read_input_token_cost": 1.45e-07,
@ -65301,6 +65507,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"fireworks_ai/kimi-latest": {
"alias_of": "fireworks_ai/accounts/fireworks/models/kimi-k3"
},
"fireworks_ai/deepseek-v4-flash-0731": {
"cache_read_input_token_cost": 7e-09,
"cache_read_input_token_cost_priority": 8.75e-09,
@ -68974,6 +69183,9 @@
"output_cost_per_token": 4.7e-07,
"source": "https://api.together.ai/v1/models"
},
"together_ai/qwen-flash-latest": {
"alias_of": "together_ai/Qwen/Qwen3.8-Flash"
},
"together_ai/moonshotai/Kimi-K2.5-fp4": {
"input_cost_per_token": 5e-07,
"output_cost_per_token": 2.8e-06,
@ -69190,6 +69402,9 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"azure_ai/glm-latest": {
"alias_of": "azure_ai/FW-GLM-5.3"
},
"azure_ai/FW-GLM-5.3-Flash": {
"cache_read_input_token_cost": 3.8e-08,
"input_cost_per_token": 1.88e-07,
@ -69205,6 +69420,9 @@
"supports_reasoning": true,
"supports_tool_choice": true
},
"azure_ai/glm-flash-latest": {
"alias_of": "azure_ai/FW-GLM-5.3-Flash"
},
"azure_ai/FW-GPT-OSS-120B": {
"deprecation_date": "2027-07-01",
"cache_read_input_token_cost": 8.2e-08,
@ -71210,6 +71428,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/glm-latest": {
"alias_of": "baseten/zai-org/GLM-5.3"
},
"baseten/zai-org/GLM-5.3-Fast": {
"cache_read_input_token_cost": 2.1e-07,
"input_cost_per_token": 2.1e-06,
@ -71553,6 +71774,13 @@
"output_cost_per_token": 1.7e-06,
"source": "https://api.together.ai/v1/models"
},
"together_ai/essentialai/rnj-1-instruct": {
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"mode": "chat",
"output_cost_per_token": 1.5e-07,
"source": "https://www.together.ai/models/rnj-1-instruct"
},
"together_ai/google/gemma-4-31B-it": {
"deprecation_date": "2026-09-15",
"deprecation_source": "https://docs.together.ai/docs/deprecations",
@ -74399,6 +74627,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/kimi-latest": {
"alias_of": "global.moonshotai.kimi-k3"
},
"us.moonshotai.kimi-k3": {
"supports_regex_lookaround": false,
"cache_creation_input_token_cost": 4.125e-06,
@ -74949,6 +75180,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/kimi-latest": {
"alias_of": "baseten/moonshotai/Kimi-K3"
},
"baseten/nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B": {
"cache_read_input_token_cost": 1.2e-07,
"input_cost_per_token": 6e-07,
@ -75031,6 +75265,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"baseten/glm-flash-latest": {
"alias_of": "baseten/zai-org/GLM-5.3-Flash"
},
"baseten/deepseek-ai/DeepSeek-V4-Flash-0731": {
"cache_read_input_token_cost": 2.8e-08,
"input_cost_per_token": 1.3e-07,
@ -75047,6 +75284,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"baseten/deepseek-flash-latest": {
"alias_of": "baseten/deepseek-ai/DeepSeek-V4-Flash-0731"
},
"baseten/deepseek-ai/DeepSeek-V4-Pro": {
"deprecation_date": "2026-10-16",
"cache_read_input_token_cost": 1.45e-07,
@ -75080,6 +75320,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"baseten/deepseek-pro-latest": {
"alias_of": "baseten/deepseek-ai/DeepSeek-V4-Pro-0813"
},
"baseten/zai-org/GLM-5.2-Fast": {
"cache_read_input_token_cost": 2.1e-07,
"input_cost_per_token": 2.1e-06,
@ -76071,6 +76314,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://management.azure.com/subscriptions/c873328e-b572-4770-8dff-aaeb6f1f0e79/providers/Microsoft.CognitiveServices/locations/eastus2/models?api-version=2024-10-01"
},
"azure_ai/claude-sonnet-latest": {
"alias_of": "azure_ai/claude-sonnet-5-5"
},
"bedrock/us-gov-east-1/anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -76235,6 +76481,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock/claude-sonnet-latest": {
"alias_of": "global.anthropic.claude-sonnet-5-5"
},
"jp.anthropic.claude-sonnet-5-5": {
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 2.75e-06,
@ -76494,6 +76743,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/claude-sonnet-latest": {
"alias_of": "vertex_ai/claude-sonnet-5-5"
},
"vertex_ai/claude-sonnet-5-5@default": {
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
@ -76606,6 +76858,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock/grok-latest": {
"alias_of": "global.xai.grok-4.7"
},
"us.xai.grok-4.7": {
"supports_regex_lookaround": false,
"supported_endpoints": ["/v1/chat/completions"],
@ -76778,6 +77033,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-sol-latest": {
"alias_of": "gpt-6.1-sol"
},
"global.openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
@ -76824,6 +77082,9 @@
"supports_sampling_params": false,
"supports_xhigh_reasoning_effort": true
},
"bedrock/gpt-sol-latest": {
"alias_of": "global.openai.gpt-6.1-sol"
},
"bedrock_mantle/openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
@ -76872,6 +77133,9 @@
"supports_xhigh_reasoning_effort": true,
"use_openai_responses_path": true
},
"bedrock_mantle/gpt-sol-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6.1-sol"
},
"us.openai.gpt-6.1-sol": {
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
@ -76963,6 +77227,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"vertex_ai/grok-latest": {
"alias_of": "vertex_ai/xai/grok-4.7"
},
"azure_ai/kimi-k2-thinking": {
"deprecation_date": "2026-03-29",
"deprecation_source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
@ -77020,6 +77287,9 @@
"supports_tool_choice": true,
"supports_vision": false
},
"bedrock/glm-latest": {
"alias_of": "global.zai.glm-5.3"
},
"us.zai.glm-5.3": {
"cache_creation_input_token_cost": 2.31e-06,
"cache_creation_input_token_cost_batches": 1.155e-06,
@ -77243,6 +77513,9 @@
"supports_vision": true,
"supports_web_search": true
},
"azure_ai/grok-latest": {
"alias_of": "azure_ai/grok-4.7"
},
"in.moonshotai.kimi-k3": {
"cache_creation_input_token_cost": 4.125e-06,
"cache_creation_input_token_cost_flex": 2.0625e-06,
@ -77329,6 +77602,9 @@
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08
},
"claude-haiku-latest": {
"alias_of": "claude-haiku-5-5"
},
"bedrock_mantle/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
@ -77375,6 +77651,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"bedrock_mantle/claude-haiku-latest": {
"alias_of": "bedrock_mantle/anthropic.claude-haiku-5-5"
},
"bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
@ -77606,6 +77885,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
},
"azure_ai/claude-haiku-latest": {
"alias_of": "azure_ai/claude-haiku-5-5"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
@ -77796,6 +78078,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"bedrock/claude-haiku-latest": {
"alias_of": "global.anthropic.claude-haiku-5-5"
},
"jp.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
@ -78003,6 +78288,9 @@
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"vertex_ai/claude-haiku-latest": {
"alias_of": "vertex_ai/claude-haiku-5-5"
},
"vertex_ai/claude-haiku-5-5@default": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
@ -78907,6 +79195,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/grok-latest": {
"alias_of": "github_copilot/grok-4.7"
},
"together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
"cache_read_input_token_cost": 6e-09,
"input_cost_per_token": 3e-07,
@ -80330,6 +80621,9 @@
"supports_web_search": true,
"supports_xhigh_reasoning_effort": true
},
"gpt-terra-latest": {
"alias_of": "gpt-6.1-terra"
},
"chatgpt/gpt-6.1-terra": {
"litellm_provider": "chatgpt",
"max_input_tokens": 922000,
@ -80388,6 +80682,9 @@
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"bedrock/gpt-terra-latest": {
"alias_of": "global.openai.gpt-6.1-terra"
},
"us.openai.gpt-6.1-terra": {
"cache_creation_input_token_cost": 1.375e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.75e-06,
@ -80467,6 +80764,9 @@
"supports_response_schema": true,
"supports_vision": true
},
"github_copilot/gpt-terra-latest": {
"alias_of": "github_copilot/gpt-6.1-terra"
},
"refuseless/glm-5.3-abliterated": {
"cache_read_input_token_cost": 1.5e-07,
"input_cost_per_token": 1e-06,
@ -80621,6 +80921,9 @@
"supports_xhigh_reasoning_effort": true,
"use_openai_responses_path": true
},
"bedrock_mantle/gpt-terra-latest": {
"alias_of": "bedrock_mantle/openai.gpt-6.1-terra"
},
"clf_ai_gateway/glm-5.3": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
@ -82170,6 +82473,9 @@
"supports_vision": true,
"thinking_always_on": false
},
"databricks/claude-haiku-latest": {
"alias_of": "databricks/databricks-claude-haiku-5-5"
},
"databricks/databricks-claude-sonnet-5-5": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_creation_input_token_cost_above_1hr": 4.00001e-06,
@ -82199,6 +82505,9 @@
"supports_vision": false,
"thinking_always_on": true
},
"databricks/claude-sonnet-latest": {
"alias_of": "databricks/databricks-claude-sonnet-5-5"
},
"databricks/databricks-deepseek-v4-1-flash": {
"cache_creation_input_token_cost": 3.0002e-07,
"cache_read_input_token_cost": 3.003e-08,
@ -82431,6 +82740,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 3.9998e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 9.99999e-06
},
"databricks/gpt-sol-latest": {
"alias_of": "databricks/databricks-gpt-6-1-sol"
},
"databricks/databricks-gpt-6-1-terra": {
"cache_creation_input_token_cost": 1.24999e-06,
"cache_creation_input_token_cost_above_272k_tokens": 2.49998e-06,
@ -82471,6 +82783,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 1.9999e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.00003e-06
},
"databricks/gpt-terra-latest": {
"alias_of": "databricks/databricks-gpt-6-1-terra"
},
"databricks/databricks-gpt-6-astra": {
"cache_creation_input_token_cost": 1.249997e-05,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
@ -82511,6 +82826,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 4.00001e-06,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.000002e-05
},
"databricks/gpt-astra-latest": {
"alias_of": "databricks/databricks-gpt-6-astra"
},
"databricks/databricks-gpt-6-luna": {
"cache_creation_input_token_cost": 1.2502e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.4997e-07,
@ -82551,6 +82869,9 @@
"cache_read_input_token_cost_above_272k_tokens_priority": 3.997e-08,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5.0001e-07
},
"databricks/gpt-luna-latest": {
"alias_of": "databricks/databricks-gpt-6-luna"
},
"databricks/databricks-gpt-6-sol": {
"cache_creation_input_token_cost": 2.49998e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.00003e-06,
@ -82681,6 +83002,9 @@
"supports_tool_choice": true,
"supports_vision": true
},
"databricks/grok-latest": {
"alias_of": "databricks/databricks-grok-4-7"
},
"databricks/databricks-kimi-k2-7": {
"cache_creation_input_token_cost": 9.4997e-07,
"cache_read_input_token_cost": 1.8998e-07,
@ -86613,6 +86937,9 @@
"supports_vision": true,
"use_openai_responses_path": true
},
"bedrock_mantle/grok-latest": {
"alias_of": "bedrock_mantle/xai.grok-4.7"
},
"together_ai/ricursive-ai/Ripple-65B-Preview": {
"cache_read_input_token_cost": 2e-08,
"input_cost_per_token": 2e-07,

2
tests/e2e/AGENTS.md
View file

@ -50,7 +50,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `gateway/` - proxy configuration only (`litellm-config.yml`). No tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture. Ships its own driver/builder/publisher, covered by `tests/e2e_harness/claude_code/`. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`). The CLI-driving path stays bespoke. `pr_gate_version_resolver.py` only forwards to `cli_harness/npm_version_resolver.py`, because the Buildkite runner images still run that path and install @latest without it
- `cli_harness/` - no tests, only the plumbing every suite that drives a real coding-agent CLI shares: `npm_version_resolver.py` picks the CLI release to install (the newest npm release at least three days old, run by CI as `python tests/e2e/cli_harness/npm_version_resolver.py <npm package>`, stdlib only), `lockdown.py` gives the CLI subprocess an allowlisted env and a fresh empty HOME so it never sees the runner's credentials or dotfiles, and `rate_limiter.py` is the cross-process token bucket per upstream that keeps parallel CLI runs under one request rate. Covered by `tests/e2e_harness/cli_harness/`. Code only one CLI suite needs stays in that suite
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
- `codex/` - the Codex compatibility suite: drives the real `codex exec --json` CLI through the gateway, one class per row (the command round trip, a file edit with the patch tool, a long session that follows a chain of files one command at a time, auto-compaction, where Codex reads a random value, compacts through the gateway and is asked for the value after the file is deleted, and an MCP tool, where Codex calls the tool of `mcp_server.py`, a stdio MCP server the test hands it with `add_mcp_server`, on a first and a resumed turn, and a sub-agent, where Codex's multi-agent v2 hands a helper a random codeword as its task with shell tools off so the helper can only learn it from the task, and the helper's own final reply is read from its rollout), each row parametrized over `SETUPS` in `setups.py`, one `pytest.param` per setup (GPT on Bedrock Mantle, Bedrock Runtime, Azure OpenAI and OpenAI, plus Claude on Bedrock). A row with one setup broken by a known bug parametrizes over `with_expected_failure(<setup id>, reason=...)`, which is `SETUPS` with a strict xfail on that one setup. The `setup` fixture registers that setup's deployment under a fresh name via `/model/new` and the `codex` fixture makes a key scoped to it, and both are deleted on teardown. `harness.py` gives each test a throwaway HOME and CODEX_HOME whose `config.toml` points a custom `responses` provider at the gateway and has Codex read its model list from the gateway's `/v1/models` (`model_catalog_url` on the provider), with the sandbox on (`workspace-write`) and approvals off, and runs the CLI under `cli_harness`'s env lockdown and rate limiter. `run.py` reads the JSONL events into a `CodexRun`: the commands with their output, the patched files, the MCP tool calls with the text they returned, the last agent message as `answer`, the usage and the errors, where an error's `status_code` is what lets the suite's 5xx rerun rule catch a turn Codex gave up on. `codex exec --json` prints nothing when Codex compacts, so `rollout.py` reads the compactions from the thread's rollout file under CODEX_HOME/sessions, and a compaction is remote when its replacement history kept the provider's `compaction` item. `subagents` reads the sub-agents a thread spawned from their own rollouts, whose `session_meta` names that thread as their parent, each with its final replies and whether it forked the parent's conversation (`forked_from_id`), so the sub-agent row can reject a helper that saw the codeword in inherited history instead of its task. `exec` takes `enable` and `disable`, Codex feature names it passes as `--enable` and `--disable`. `exec` and `resume` take `compact_after_tokens`, which forces Codex's remote compaction (`compaction_trigger` items over `/v1/responses`, the path Codex otherwise only takes for OpenAI and Azure) on the gateway provider and compacts once the context grows past that many tokens. A test passes on a random value the model can only learn through the gateway, never on the model's wording or the exact command it typed. It needs `codex` on PATH and fails without it. Covered by `tests/e2e_harness/codex/`
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream, see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport. The Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`. Its specs follow the dashboard's rule against testing what shadcn and Base UI already guarantee, in `ui/litellm-dashboard/AGENTS.md`
## MCP suite: real Datadog only

39
tests/e2e/codex/harness.py
View file

@ -12,7 +12,7 @@ from typing import Final
from cli_harness.lockdown import isolated_home, locked_down_env
from cli_harness.rate_limiter import RateBucket, RateLimiter
from codex.rollout import Compaction, rollout_compactions
from codex.rollout import Compaction, Subagent, rollout_compactions, rollout_parent_thread, rollout_subagent
from codex.run import CodexRun, codex_run
from e2e_metadata import step
@ -71,8 +71,28 @@ class CodexHarness:
_ = config.write(_mcp_server_table(name, server))
@step('Run Codex on {model} through the proxy with the prompt "{prompt}"')
def exec(self, prompt: str, *, model: str, cwd: Path, compact_after_tokens: int | None = None) -> CodexRun:
return self._run(("exec",), (*_compaction_flags(compact_after_tokens), "--model", model, "--", prompt), cwd)
def exec(
self,
prompt: str,
*,
model: str,
cwd: Path,
compact_after_tokens: int | None = None,
enable: tuple[str, ...] = (),
disable: tuple[str, ...] = (),
) -> CodexRun:
return self._run(
("exec",),
(
*_compaction_flags(compact_after_tokens),
*_feature_flags(enable=enable, disable=disable),
"--model",
model,
"--",
prompt,
),
cwd,
)
@step('Resume Codex thread {thread_id} on {model} through the proxy with the prompt "{prompt}"')
def resume(
@ -88,6 +108,12 @@ class CodexHarness:
rollouts: Final = sorted(self.codex_home.glob(f"sessions/**/rollout-*-{thread_id}.jsonl"))
return tuple(chain.from_iterable(rollout_compactions(rollout.read_text()) for rollout in rollouts))
def subagents(self, thread_id: str) -> tuple[Subagent, ...]:
rollouts: Final = tuple(
path.read_text() for path in sorted(self.codex_home.glob("sessions/**/rollout-*.jsonl"))
)
return tuple(rollout_subagent(rollout) for rollout in rollouts if rollout_parent_thread(rollout) == thread_id)
def _run(self, subcommand: tuple[str, ...], args: tuple[str, ...], cwd: Path) -> CodexRun:
_ = self.limiter.acquire(self.rate_bucket)
completed: Final = subprocess.run(
@ -116,6 +142,13 @@ def _mcp_server_table(name: str, server: McpServer) -> str:
)
def _feature_flags(*, enable: tuple[str, ...], disable: tuple[str, ...]) -> tuple[str, ...]:
return (
*chain.from_iterable(("--enable", feature) for feature in enable),
*chain.from_iterable(("--disable", feature) for feature in disable),
)
def _compaction_flags(after_tokens: int | None) -> tuple[str, ...]:
match after_tokens:
case None:

62
tests/e2e/codex/rollout.py
View file

@ -1,19 +1,23 @@
"""The compactions Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-<thread id>.jsonl`.
"""What Codex recorded in a thread's rollout file, `CODEX_HOME/sessions/**/rollout-*-<thread id>.jsonl`.
`codex exec --json` prints no event for a compaction, so the rollout is the only place it shows. Each line is
`{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`) and only `compacted` lines are read.
`codex exec --json` prints no event for a compaction or for a sub-agent's own turn, so the rollout is the only place
they show. Each line is `{"timestamp", "type", "payload"}` (`codex-rs/history/src/rollout_payload.rs`).
A remote compaction is one whose replacement history carries the provider's `compaction` item, while a local one
replaces the history with a summary Codex wrote itself.
replaces the history with a summary Codex wrote itself. A sub-agent gets a rollout of its own whose `session_meta`
names the thread that spawned it, and carries `forked_from_id` when it inherited that thread's conversation. Each turn
it finishes ends in a `task_complete` event with its last message.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Final
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict
_COMPACTED: Final = "compacted"
_SESSION_META: Final = "session_meta"
_EVENT: Final = "event_msg"
_PROVIDER_COMPACTION_ITEM: Final = "compaction"
@ -34,14 +38,58 @@ class _CompactedPayload(_Frozen):
replacement_history: tuple[_HistoryItem, ...] | None = None
class _SessionMeta(_Frozen):
parent_thread_id: str | None = None
forked_from_id: str | None = None
class _Event(_Frozen):
type: str
class _TaskComplete(_Frozen):
type: Literal["task_complete"]
last_agent_message: str | None = None
@dataclass(frozen=True, slots=True)
class Compaction:
remote: bool
@dataclass(frozen=True, slots=True)
class Subagent:
forked: bool
replies: tuple[str, ...]
def rollout_compactions(rollout: str) -> tuple[Compaction, ...]:
lines: Final = tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip())
return tuple(_compaction(line.payload) for line in lines if line.type == _COMPACTED)
return tuple(_compaction(line.payload) for line in _lines(rollout) if line.type == _COMPACTED)
def rollout_parent_thread(rollout: str) -> str | None:
return _session_meta(rollout).parent_thread_id
def rollout_subagent(rollout: str) -> Subagent:
return Subagent(forked=_session_meta(rollout).forked_from_id is not None, replies=rollout_final_messages(rollout))
def rollout_final_messages(rollout: str) -> tuple[str, ...]:
events: Final = (line.payload for line in _lines(rollout) if line.type == _EVENT)
completions: Final = (
_TaskComplete.model_validate(event) for event in events if _Event.model_validate(event).type == "task_complete"
)
return tuple(done.last_agent_message for done in completions if done.last_agent_message is not None)
def _session_meta(rollout: str) -> _SessionMeta:
meta: Final = next((line.payload for line in _lines(rollout) if line.type == _SESSION_META), None)
return _SessionMeta.model_validate(meta) if meta is not None else _SessionMeta()
def _lines(rollout: str) -> tuple[_Line, ...]:
return tuple(_Line.model_validate_json(line) for line in rollout.splitlines() if line.strip())
def _compaction(payload: object) -> Compaction:

31
tests/e2e/codex/test_subagent_e2e.py Normal file
View file

@ -0,0 +1,31 @@
from __future__ import annotations
from typing import Final
import pytest
from codex.harness import CodexHarness
from codex.setups import SETUPS, ModelSetup
from e2e_config import unique_marker
pytestmark = pytest.mark.e2e
@pytest.mark.parametrize("deployment", SETUPS)
class TestCodexSubagent:
def test_a_subagent_reads_the_task_its_parent_hands_it(self, codex: CodexHarness, setup: ModelSetup) -> None:
codeword: Final = unique_marker()
run: Final = codex.exec(
'Spawn exactly one sub-agent with the spawn_agent tool, with fork_turns set to "none" so it does not'
f" inherit this conversation. Its task: reply with exactly this codeword and nothing else: {codeword}."
" Wait for its reply, then tell me the exact reply it sent you, verbatim.",
model=setup.model,
cwd=codex.new_workspace(files={}),
enable=("multi_agent_v2",),
disable=("shell_tool", "unified_exec"),
)
assert run.exit_code == 0, run
assert run.thread_id is not None, run
subagents: Final = codex.subagents(run.thread_id)
assert subagents, run
assert not any(subagent.forked for subagent in subagents), (subagents, run)
assert any(any(codeword in reply for reply in subagent.replies) for subagent in subagents), (subagents, run)

87
tests/e2e_harness/codex/test_codex_rollout.py
View file

@ -8,7 +8,14 @@ from typing import Final
from cli_harness.rate_limiter import RateBucket, RateLimiter
from codex.harness import CodexHarness
from codex.rollout import Compaction, rollout_compactions
from codex.rollout import (
Compaction,
Subagent,
rollout_compactions,
rollout_final_messages,
rollout_parent_thread,
rollout_subagent,
)
THREAD_ID: Final = "01a18fea-b2e8-7ed2-a29b-1ce4d2b138c3"
USER_PROMPT: Final = {
@ -60,13 +67,83 @@ class TestRolloutCompactions:
assert rollout_compactions(_rollout()) == ()
def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None:
codex: Final = CodexHarness(
home=tmp_path,
def _harness(home: Path) -> CodexHarness:
return CodexHarness(
home=home,
api_key="unused",
rate_bucket=RateBucket.OPENAI,
limiter=RateLimiter({}, tmp_path, time.monotonic, time.sleep),
limiter=RateLimiter({}, home, time.monotonic, time.sleep),
)
def _subagent_rollout(
*, thread_id: str, parent: str | None, final_messages: tuple[str | None, ...], forked: bool = False
) -> str:
meta: Final = (
{"id": thread_id, "cli_version": "0.165.0"}
| ({"parent_thread_id": parent} if parent else {})
| ({"forked_from_id": parent} if forked else {})
)
inherited: Final = (_line("session_meta", {"id": parent, "cli_version": "0.165.0"}),) if forked else ()
return "\n".join(
(
_line("session_meta", meta),
*inherited,
_line("event_msg", {"type": "task_started", "turn_id": "t1"}),
_line(
"response_item",
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "thinking"}]},
),
*(
_line("event_msg", {"type": "task_complete", "turn_id": "t1", "last_agent_message": message})
for message in final_messages
),
)
)
class TestRolloutSubagents:
def test_a_subagent_rollout_names_the_thread_that_spawned_it(self) -> None:
rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=())
assert rollout_parent_thread(rollout) == THREAD_ID
def test_a_top_level_rollout_has_no_parent(self) -> None:
assert rollout_parent_thread(_rollout()) is None
def test_a_subagent_that_inherited_its_parents_conversation_is_forked(self) -> None:
rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",), forked=True)
assert rollout_subagent(rollout) == Subagent(forked=True, replies=("hi",))
def test_a_subagent_that_started_fresh_is_not_forked(self) -> None:
rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("hi",))
assert rollout_subagent(rollout) == Subagent(forked=False, replies=("hi",))
def test_the_final_messages_are_the_last_message_of_each_finished_turn(self) -> None:
rollout: Final = _subagent_rollout(thread_id="child", parent=THREAD_ID, final_messages=("first", None, "last"))
assert rollout_final_messages(rollout) == ("first", "last")
def test_the_harness_reads_only_the_subagents_of_the_thread_it_is_asked_about(tmp_path: Path) -> None:
codex: Final = _harness(tmp_path)
sessions: Final = codex.codex_home / "sessions" / "2026" / "11" / "06"
sessions.mkdir(parents=True)
rollouts: Final = {
THREAD_ID: _subagent_rollout(thread_id=THREAD_ID, parent=None, final_messages=("the parent's answer",)),
"child-a": _subagent_rollout(thread_id="child-a", parent=THREAD_ID, final_messages=("reply a",)),
"child-b": _subagent_rollout(thread_id="child-b", parent=THREAD_ID, final_messages=("reply b",), forked=True),
"stranger": _subagent_rollout(thread_id="stranger", parent="other-thread", final_messages=("not ours",)),
}
for thread_id, rollout in rollouts.items():
_ = (sessions / f"rollout-2026-11-06T17-01-05-{thread_id}.jsonl").write_text(rollout)
assert codex.subagents(THREAD_ID) == (
Subagent(forked=False, replies=("reply a",)),
Subagent(forked=True, replies=("reply b",)),
)
def test_the_harness_reads_only_the_rollout_of_the_thread_it_is_asked_about(tmp_path: Path) -> None:
codex: Final = _harness(tmp_path)
sessions: Final = codex.codex_home / "sessions" / "2026" / "10" / "30"
sessions.mkdir(parents=True)
_ = (sessions / f"rollout-2026-10-30T20-03-32-{THREAD_ID}.jsonl").write_text(_rollout(REMOTE))

61
tests/integration/providers/test_bedrock_gpt_responses_native_wire.py
View file

@ -1,14 +1,15 @@
import base64
import json
import uuid
from dataclasses import dataclass
from typing import Final
import openai
from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, respond, target_of
from integration._support.bedrock_runtime_peer import NATIVE_RESPONSES, answer, marker_of, respond, target_of
from integration._support.client import Gateway, Scenario, eventually
from integration._support.database import read_rows
from integration._support.wire import Request, Wire, wire_server
from openai.types.responses import ResponseCompletedEvent, ResponseTextDeltaEvent
from integration._support.wire import Reply, Request, Wire, wire_server
from openai.types.responses import ResponseCompletedEvent, ResponseFailedEvent, ResponseTextDeltaEvent
from pydantic import JsonValue, TypeAdapter
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with
@ -17,6 +18,7 @@ GPT: Final = "us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
SALT: Final = "sk-integration-salt"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
UPSTREAM_OUTAGE: Final = "The server had an error while processing your request. Sorry about that!"
@dataclass(frozen=True, slots=True)
@ -123,6 +125,59 @@ async def test_async_openai_sdk_responses_stream_is_served_by_the_native_respons
assert _spend_row(completed.response.id) == _success_row(model)
def _stream_then_bare_error(request: Request) -> Reply:
marker: Final = marker_of(request)
created: Final[dict[str, JsonValue]] = {
"type": "response.created",
"sequence_number": 0,
"response": {
"id": f"resp_upstream_{marker}",
"object": "response",
"created_at": 1,
"status": "in_progress",
"model": GPT,
"output": [],
},
}
delta: Final[dict[str, JsonValue]] = {
"type": "response.output_text.delta",
"sequence_number": 1,
"item_id": f"msg_{marker}",
"output_index": 0,
"content_index": 0,
"delta": answer(marker),
}
outage: Final[dict[str, JsonValue]] = {
"error": {"message": UPSTREAM_OUTAGE, "type": "server_error", "code": "service_unavailable"}
}
return Reply(
content_type="text/event-stream",
chunks=tuple(f"data: {json.dumps(event)}\n\n".encode() for event in (created, delta, outage)),
)
async def test_mid_stream_upstream_error_without_an_event_type_reaches_the_client_as_response_failed(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_stream_then_bare_error) as wire, gateway.scenario() as scenario:
model: Final = _deployment(scenario, wire)
client: Final = openai.AsyncOpenAI(base_url=f"{gateway.client.base_url}/v1", api_key=gateway.key, max_retries=0)
stream: Final = await client.responses.create(
model=model, input=_prompt(marker), stream=True, extra_body={"cache": {"no-cache": True}}
)
events: Final = [event async for event in stream]
assert [event.type for event in events] == [
"response.created",
"response.output_text.delta",
"response.failed",
], events
failed: Final = events[-1]
assert isinstance(failed, ResponseFailedEvent), failed
assert failed.response.error is not None and UPSTREAM_OUTAGE in failed.response.error.message, failed
_native_request(wire)
def test_codex_agent_message_with_encrypted_content_reaches_the_native_responses_route_intact(
gateway: Gateway,
) -> None:

8
tests/integration/spend/test_normalized_error_long_message.py
View file

@ -358,16 +358,17 @@ def _burst(
*,
rejected_model: str = CRAFTED_MODEL,
) -> tuple[_BurstResult, ...]:
nonce: Final = uuid.uuid4().hex
crafted: Final = tuple(
(f"crafted-{path}-{index}", path, _body(path, rejected_model, uuid.uuid4().hex, stream=index % 2 == 1), open_key)
for path in (CHAT, MESSAGES, RESPONSES)
for index in range(4)
)
happy: Final = tuple(
(f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}"), happy_key) for index in range(8)
(f"happy-{index}", CHAT, _body(CHAT, happy_model, f"happy-{index}-{nonce}"), happy_key) for index in range(8)
)
late: Final = tuple(
(f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}"), happy_key) for index in range(8)
(f"late-{index}", CHAT, _body(CHAT, happy_model, f"late-{index}-{nonce}"), happy_key) for index in range(8)
)
with (
ThreadPoolExecutor(max_workers=28) as pool,
@ -427,6 +428,7 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy
open_key: Final = scenario.key()
control: Final = _burst(client, model, key, open_key, lambda: None, rejected_model=BENIGN_288KB_MODEL)
assert all(result.status == (400 if result.label.startswith("crafted") else 200) for result in control), control
wire.drain()
workers: Final = owned.worker_pids()
assert len(workers) == 2, workers
victim: Final = min(workers)
@ -438,6 +440,8 @@ def test_killing_one_worker_mid_burst_leaves_the_other_serving_crafted_and_happy
_assert_burst_rejections_fast(results, control)
late: Final = tuple(result for result in results if result.label.startswith("late"))
assert all(result.status == 200 for result in late), late
seen: Final = tuple(request.body.decode() for request in wire.drain())
assert all(any(f"normalized error audit {result.label}-" in body for body in seen) for result in late), seen
_assert_rows_land_exactly_once(results, "late")
survivor: Final = tuple(pid for pid in owned.worker_pids() if pid != victim)
assert survivor, "no worker left serving"

380
tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -3837,6 +3837,386 @@ def test_reasoning_items_streaming_emitted_on_response_completed():
assert ri["summary"][0]["text"] == summary_text
def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keeps():
"""A chatgpt /v1/messages stream drops the reasoning item unless this delta carries thinking_blocks.
response.output_item.done is the only event that has encrypted_content. The chat delta has to
expose it as a signed thinking block, and that block alone has to count as stream content.
"""
from unittest.mock import MagicMock
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
encrypted: Final = "gAAAA_round_trip"
done: Final = {
"type": "response.output_item.done",
"output_index": 0,
"item": {
"type": "reasoning",
"id": "rs_done",
"encrypted_content": encrypted,
"summary": [],
},
}
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True, streams_reasoning_signatures=True
).chunk_parser(done)
blocks: Final = result.choices[0].delta.thinking_blocks
assert blocks is not None
assert blocks[0]["type"] == "redacted_thinking"
assert blocks[0]["data"] == encrypted_reasoning_signature(encrypted)
assert blocks[0]["index"] == 0
assert getattr(result.choices[0].delta, "reasoning_content", None) in (None, "")
wrapper: Final = CustomStreamWrapper(
completion_stream=None,
model="chatgpt/gpt-6.1-sol",
logging_obj=MagicMock(),
custom_llm_provider="chatgpt",
)
assert wrapper.is_chunk_non_empty({}, result, {}) is True
block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block(
choices=result.choices
)
assert block_type == "redacted_thinking"
assert start["data"] == encrypted_reasoning_signature(encrypted)
_SIGNED_REASONING_STREAM_BODY: Final[Mapping[str, object]] = {
**_searched_responses_body("azure/gpt-6-sol"),
"output": [
{"type": "reasoning", "id": "rs_signed", "summary": [], "encrypted_content": "gAAAA_consumer"},
_PONG_MESSAGE,
],
}
def _signed_reasoning_sse() -> str:
events: Final = (
{
"type": "response.created",
"response": {**_SIGNED_REASONING_STREAM_BODY, "status": "in_progress", "output": [], "usage": None},
},
{"type": "response.output_item.done", "output_index": 0, "item": _SIGNED_REASONING_STREAM_BODY["output"][0]},
{
"type": "response.output_text.delta",
"item_id": "msg_logged_call_type",
"output_index": 1,
"content_index": 0,
"delta": "pong",
},
{"type": "response.completed", "response": _SIGNED_REASONING_STREAM_BODY},
)
return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events)
_AZURE_BRIDGE_KWARGS: Final[Mapping[str, object]] = {
"api_base": "https://example.invalid",
"api_version": "2025-04-01-preview",
"api_key": "test-key",
}
@pytest.mark.asyncio
async def test_streamed_reasoning_signature_reaches_messages_clients_and_never_chat_completions_deltas(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
with respx.mock(assert_all_called=True) as router:
router.post(url__startswith="https://example.invalid/openai/responses").mock(
side_effect=lambda _request: httpx.Response(
200, text=_signed_reasoning_sse(), headers={"content-type": "text/event-stream"}
)
)
chat_stream: Final = await litellm.acompletion(
model="azure/gpt-6-sol",
messages=[{"role": "user", "content": "ping"}],
tools=[_GET_WEATHER_TOOL],
stream=True,
**_AZURE_BRIDGE_KWARGS,
)
chat_deltas: Final = [chunk.choices[0].delta async for chunk in chat_stream if chunk.choices]
messages_stream: Final = await litellm.anthropic.messages.acreate(
model="azure/gpt-6-sol",
messages=[{"role": "user", "content": "ping"}],
tools=[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
max_tokens=64,
stream=True,
**_AZURE_BRIDGE_KWARGS,
)
sse: Final = b"".join([chunk async for chunk in messages_stream]).decode()
assert [getattr(delta, "thinking_blocks", None) for delta in chat_deltas if getattr(delta, "thinking_blocks", None)] == []
assert "".join(delta.content or "" for delta in chat_deltas) == "pong"
messages_events: Final = [json.loads(line[len("data: ") :]) for line in sse.splitlines() if line.startswith("data: ")]
started: Final = [event["content_block"] for event in messages_events if event.get("type") == "content_block_start"]
assert [block["type"] for block in started] == ["redacted_thinking", "text"]
assert started[0]["data"] == encrypted_reasoning_signature("gAAAA_consumer")
def test_reasoning_output_item_done_with_summary_signs_the_thinking_text():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
encrypted: Final = "gAAAA_with_summary"
done: Final = {
"type": "response.output_item.done",
"output_index": 2,
"item": {
"type": "reasoning",
"id": "rs_sum",
"encrypted_content": encrypted,
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
},
}
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True, streams_reasoning_signatures=True
).chunk_parser(done)
block: Final = result.choices[0].delta.thinking_blocks[0]
assert block["type"] == "thinking"
assert block["thinking"] == "Oslo is colder."
assert block["signature"] == encrypted_reasoning_signature(encrypted)
assert block["index"] == 2
def test_streamed_reasoning_summary_reaches_messages_clients_once_with_its_signature():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicStreamWrapper
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
encrypted: Final = "gAAAA_streamed_summary"
events: Final = (
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0},
{"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "Oslo is "},
{"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "colder."},
{
"type": "response.output_item.done",
"output_index": 0,
"item": {
"type": "reasoning",
"id": "rs_streamed",
"encrypted_content": encrypted,
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
},
},
{"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."},
)
parser: Final = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True, streams_reasoning_signatures=True
)
chunks: Final = (
*(parser.chunk_parser(event) for event in events),
ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
)
sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="chatgpt/gpt-5.5"))
deltas: Final = [event["delta"] for event in sse_events if event.get("type") == "content_block_delta"]
assert "".join(d.get("thinking", "") for d in deltas if d["type"] == "thinking_delta") == "Oslo is colder."
assert [d["signature"] for d in deltas if d["type"] == "signature_delta"] == [
encrypted_reasoning_signature(encrypted)
]
def test_reasoning_item_repeated_at_stream_end_reaches_messages_clients_once():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.messages.adapters.streaming_iterator import AnthropicStreamWrapper
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
encrypted: Final = "gAAAA_completed_repeat"
reasoning_item: Final = {"type": "reasoning", "id": "rs_repeat", "summary": [], "encrypted_content": encrypted}
events: Final = (
{"type": "response.output_item.done", "output_index": 0, "item": reasoning_item},
{"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."},
)
parser: Final = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True, streams_reasoning_signatures=True
)
chunks: Final = (
*(parser.chunk_parser(event) for event in events),
ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(reasoning_items=[reasoning_item]), finish_reason=None)]
),
ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
)
sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="azure/gpt-5.4-nano"))
started: Final = [event["content_block"] for event in sse_events if event.get("type") == "content_block_start"]
assert [block["type"] for block in started] == ["redacted_thinking", "text"]
assert started[0]["data"] == encrypted_reasoning_signature(encrypted)
def test_reasoning_output_item_done_without_encrypted_content_stays_empty():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
done: Final = {
"type": "response.output_item.done",
"output_index": 0,
"item": {
"type": "reasoning",
"id": "rs_plain",
"summary": [{"type": "summary_text", "text": "already streamed"}],
},
}
result: Final = OpenAiResponsesToChatCompletionStreamIterator(
streaming_response=None, sync_stream=True, streams_reasoning_signatures=True
).chunk_parser(done)
assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, [])
def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only():
from unittest.mock import Mock
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
encrypted: Final = "gAAAA_non_stream"
reasoning: Final = ResponseReasoningItem(
id="rs_tool",
summary=[],
type="reasoning",
content=None,
encrypted_content=encrypted,
status=None,
)
function_call: Final = ResponseFunctionToolCall(
id="fc_1",
type="function_call",
status="completed",
arguments='{"city":"Oslo"}',
call_id="call_1",
name="get_weather",
)
raw: Final = ResponsesAPIResponse.model_construct(
id="resp_tool",
created_at=1,
error=None,
incomplete_details=None,
model="gpt-6.1-sol",
output=[reasoning, function_call],
status="completed",
usage=None,
)
model_response: Final = ModelResponse(
id="chatcmpl-tool",
created=1,
model=None,
object="chat.completion",
system_fingerprint=None,
choices=[],
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
)
result: Final = LiteLLMResponsesTransformationHandler().transform_response(
model="gpt-6.1-sol",
raw_response=raw,
model_response=model_response,
logging_obj=Mock(),
request_data={"model": "gpt-6.1-sol"},
messages=[{"role": "user", "content": "Which city?"}],
optional_params={},
litellm_params={},
encoding=Mock(),
)
message: Final = result.choices[0].message
assert message.reasoning_items[0]["encrypted_content"] == encrypted
assert "thinking_blocks" not in result.model_dump()["choices"][0]["message"]
assert message.tool_calls[0].function.name == "get_weather"
anthropic_response: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
response=result, tool_name_mapping=None
)
content: Final = anthropic_response["content"]
assert [block["type"] for block in content] == ["redacted_thinking", "tool_use"]
assert content[0]["data"] == encrypted_reasoning_signature(encrypted)
@pytest.mark.parametrize(
("call_type", "kept_ids"),
[
pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"),
pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"),
],
)
def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item(
call_type: str, kept_ids: list[str]
) -> None:
from types import SimpleNamespace
from unittest.mock import Mock
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
def reasoning(item_id: str) -> ResponseReasoningItem:
return ResponseReasoningItem(
id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None
)
def call(index: int, city: str) -> ResponseFunctionToolCall:
return ResponseFunctionToolCall(
id=f"fc_{index}",
type="function_call",
status="completed",
arguments=json.dumps({"city": city}),
call_id=f"call_{index}",
name="get_weather",
)
raw: Final = ResponsesAPIResponse.model_construct(
id="resp_two_reasoning",
created_at=1,
error=None,
incomplete_details=None,
model="gpt-5.4-nano",
output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")],
status="completed",
usage=None,
)
result: Final = LiteLLMResponsesTransformationHandler().transform_response(
model="gpt-5.4-nano",
raw_response=raw,
model_response=ModelResponse(
id="chatcmpl-two-reasoning",
created=1,
model=None,
object="chat.completion",
system_fingerprint=None,
choices=[],
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
),
logging_obj=SimpleNamespace(call_type=call_type),
request_data={"model": "gpt-5.4-nano"},
messages=[{"role": "user", "content": "Oslo or Lyon?"}],
optional_params={},
litellm_params={"custom_llm_provider": "azure"},
encoding=Mock(),
)
message: Final = result.choices[0].message
assert [item["id"] for item in message.reasoning_items] == kept_ids
assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"]
def test_streaming_function_call_tool_id_for_degenerate_call_id():
"""In streaming, Bedrock Mantle's function_call event carries a unique ``id``
(``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that

59
tests/unit/litellm_core_utils/test_get_llm_provider_logic.py
View file

@ -30,9 +30,9 @@ CUSTOM_PROVIDER: Final = "test-onprem-llm"
@pytest.mark.parametrize(
("model", "provider"),
(("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai")),
(("openai/gpt-luna-latest", None), ("gpt-luna-latest", "openai"), ("gpt-luna-latest", None)),
)
def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str | None) -> None:
def test_latest_alias_resolves_to_the_cost_map_target(model: str, provider: str | None) -> None:
target: Final = get_model_latest_aliases()["openai"]["gpt-luna-latest"]
resolved, resolved_provider, _, api_base = get_llm_provider(
model, custom_llm_provider=provider, api_base="https://api.openai.com/v1"
@ -42,7 +42,12 @@ def test_latest_alias_resolves_to_the_bundled_target(model: str, provider: str |
@pytest.mark.parametrize(
"model",
("gemini/gemini-pro-latest", "gemini/gemini-flash-latest", "dashscope/qwen-plus-latest", "mistral/mistral-large-latest"),
(
"gemini/gemini-pro-latest",
"gemini/gemini-flash-latest",
"dashscope/qwen-plus-latest",
"mistral/mistral-large-latest",
),
)
def test_provider_native_latest_alias_passes_through(model: str) -> None:
provider, name = model.split("/", 1)
@ -153,6 +158,7 @@ def _vcr_outcome_gate(request, vcr):
yield
record_vcr_outcome(request, vcr)
@pytest.fixture(scope="function")
def isolate_litellm_state():
"""
@ -205,6 +211,7 @@ def isolate_litellm_state():
setattr(litellm, attr, original_value)
_invalidate_model_cost_lowercase_map()
_SCALAR_DEFAULTS = {
"num_retries": getattr(litellm, "num_retries", None),
"num_retries_per_request": getattr(litellm, "num_retries_per_request", None),
@ -225,6 +232,7 @@ _SCALAR_DEFAULTS = {
"api_key": getattr(litellm, "api_key", None),
}
@pytest.fixture(scope="module")
def setup_and_teardown():
"""
@ -247,12 +255,14 @@ def setup_and_teardown():
litellm.in_memory_llm_clients_cache.flush_cache()
yield
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider():
_, response, _, _ = litellm.get_llm_provider(model="anthropic.claude-v2:1")
assert response == "bedrock"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - https://github.com/BerriAI/litellm/issues/4923
model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="fireworks_ai/accounts/my-test-1234")
@ -260,17 +270,20 @@ def test_get_llm_provider_fireworks(): # tests finetuned fireworks models - htt
assert custom_llm_provider == "fireworks_ai"
assert model == "accounts/my-test-1234"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_catch_all():
_, response, _, _ = litellm.get_llm_provider(model="*")
assert response == "openai"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_gpt_instruct():
_, response, _, _ = litellm.get_llm_provider(model="gpt-3.5-turbo-instruct-0914")
assert response == "text-completion-openai"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_mistral_custom_api_base():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -281,6 +294,7 @@ def test_get_llm_provider_mistral_custom_api_base():
assert model == "mistral-large-fr"
assert api_base == "https://mistral-large-fr-ishaan.francecentral.inference.ai.azure.com/v1"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_deepseek_custom_api_base():
os.environ["DEEPSEEK_API_BASE"] = "MY-FAKE-BASE"
@ -293,6 +307,7 @@ def test_get_llm_provider_deepseek_custom_api_base():
os.environ.pop("DEEPSEEK_API_BASE")
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_vertex_ai_image_models(monkeypatch):
monkeypatch.setattr(litellm, "vertex_ai_image_models", set())
@ -310,6 +325,7 @@ def test_get_llm_provider_vertex_ai_image_models(monkeypatch):
)
assert custom_llm_provider == "vertex_ai"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_ai21_chat():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -319,6 +335,7 @@ def test_get_llm_provider_ai21_chat():
assert model == "jamba-1.5-large"
assert api_base == "https://api.ai21.com/studio/v1"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_ai21_chat_test2():
"""
@ -335,6 +352,7 @@ def test_get_llm_provider_ai21_chat_test2():
assert model == "jamba-1.5-large"
assert api_base == "https://api.ai21.com/studio/v1"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_cohere_chat_test2():
"""
@ -350,6 +368,7 @@ def test_get_llm_provider_cohere_chat_test2():
assert custom_llm_provider == "cohere_chat"
assert model == "command-r-plus-08-2024"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_azure_o1():
@ -359,6 +378,7 @@ def test_get_llm_provider_azure_o1():
assert custom_llm_provider == "azure"
assert model == "o1-mini"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_hosted_vllm_default_api_key():
from litellm.litellm_core_utils.get_llm_provider_logic import (
@ -373,6 +393,7 @@ def test_hosted_vllm_default_api_key():
)
assert dynamic_api_key == "fake-api-key"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_jina_ai():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -382,6 +403,7 @@ def test_get_llm_provider_jina_ai():
assert api_base == "https://api.jina.ai/v1"
assert model == "jina-embeddings-v3"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_hosted_vllm():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -391,6 +413,7 @@ def test_get_llm_provider_hosted_vllm():
assert model == "llama-3.1-70b-instruct"
assert dynamic_api_key == "fake-api-key"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_llamafile():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -401,6 +424,7 @@ def test_get_llm_provider_llamafile():
assert dynamic_api_key == "fake-api-key"
assert api_base == "http://127.0.0.1:8080/v1"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_watson_text():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -409,6 +433,7 @@ def test_get_llm_provider_watson_text():
assert custom_llm_provider == "watsonx_text"
assert model == "watson-text-to-speech"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_azure_global_standard_get_llm_provider():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -418,6 +443,7 @@ def test_azure_global_standard_get_llm_provider():
)
assert custom_llm_provider == "azure_ai"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_nova_bedrock_converse():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -426,6 +452,7 @@ def test_nova_bedrock_converse():
assert custom_llm_provider == "bedrock"
assert model == "amazon.nova-micro-v1:0"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_bedrock_invoke_anthropic():
model, custom_llm_provider, dynamic_api_key, api_base = litellm.get_llm_provider(
@ -434,6 +461,7 @@ def test_bedrock_invoke_anthropic():
assert custom_llm_provider == "bedrock"
assert model == "invoke/anthropic.claude-haiku-4-5-20251001-v1:0"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
@pytest.mark.parametrize("model", ["xai/grok-2-vision-latest", "grok-2-vision-latest"])
def test_xai_api_base(model):
@ -450,6 +478,7 @@ def test_xai_api_base(model):
assert api_base == "https://api.x.ai/v1"
assert dynamic_api_key == "xai-my-specialkey"
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_litellm_proxy_custom_llm_provider():
"""
@ -479,6 +508,7 @@ def test_get_litellm_proxy_custom_llm_provider():
assert key == expected_api_key
assert base == expected_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_litellm_proxy_with_args_override_env_vars():
"""
@ -510,6 +540,7 @@ def test_get_litellm_proxy_with_args_override_env_vars():
assert key == arg_api_key
assert base == arg_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_litellm_proxy_model_prefix_stripping():
"""
@ -540,6 +571,7 @@ def test_get_litellm_proxy_model_prefix_stripping():
assert key == expected_api_key
assert base == expected_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true():
"""
@ -568,6 +600,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true():
assert key == proxy_api_key
assert base == proxy_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix():
"""
@ -594,6 +627,7 @@ def test_get_llm_provider_LITELLM_PROXY_ALWAYS_true_model_prefix():
assert key == proxy_api_key
assert base == proxy_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_use_proxy_arg_true():
"""
@ -623,6 +657,7 @@ def test_get_llm_provider_use_proxy_arg_true():
assert key == proxy_api_key
assert base == proxy_api_base
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
def test_get_llm_provider_use_proxy_arg_true_with_direct_args():
"""
@ -659,6 +694,7 @@ def test_get_llm_provider_use_proxy_arg_true_with_direct_args():
assert key == arg_api_key # Should use the argument key
assert base == arg_api_base # Should use the argument base
@pytest.fixture
def shipped_generalizations():
"""Install the rules shipped in the bundled backup, then restore.
@ -682,6 +718,7 @@ def shipped_generalizations():
finally:
set_fallback_generalizations(previous)
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
class TestClaudeModelPatternMatching:
"""
@ -749,9 +786,7 @@ class TestClaudeModelPatternMatching:
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
class TestTypesafeJevPatternMatching:
def test_every_priced_jev_id_routes_to_typesafe(
self, shipped_generalizations: None
) -> None:
def test_every_priced_jev_id_routes_to_typesafe(self, shipped_generalizations: None) -> None:
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
priced_ids: Final = tuple(
@ -760,9 +795,9 @@ class TestTypesafeJevPatternMatching:
if key.startswith("typesafe/")
)
assert "jev-latest" in priced_ids
assert {
model: litellm.get_llm_provider(model=model)[1] for model in priced_ids
} == dict.fromkeys(priced_ids, "typesafe")
assert {model: litellm.get_llm_provider(model=model)[1] for model in priced_ids} == dict.fromkeys(
priced_ids, "typesafe"
)
@pytest.mark.parametrize(
"model",
@ -837,7 +872,9 @@ def test_metadata_callers_never_log_in_for_a_declared_chatgpt_deployment(
assert get_api_base("chatgpt/gpt-5.4", {"model": "chatgpt/gpt-5.4"}) == "https://chatgpt.com/backend-api/codex"
identity = _resolve_model("chatgpt/gpt-5.4", None)
assert identity is not None and identity.provider == "chatgpt"
group = Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}]).registry().group(
"codex"
group = (
Router(model_list=[{"model_name": "codex", "litellm_params": {"model": "chatgpt/gpt-5.4"}}])
.registry()
.group("codex")
)
assert group is not None and group.providers == ["chatgpt"]

39
tests/unit/litellm_core_utils/test_get_model_cost_map.py
View file

@ -130,6 +130,30 @@ def test_finalize_pops_key_and_installs_rules():
set_fallback_generalizations(previous)
def test_alias_of_rows_are_served_as_their_target_and_dangling_ones_are_dropped(
caplog: pytest.LogCaptureFixture,
) -> None:
raw: Final = {
"real": {"litellm_provider": "openai", "mode": "chat", "input_cost_per_token": 1e-6},
"other": {"litellm_provider": "openai", "mode": "chat", "aliases": ["twin-latest"]},
"real-latest": {"alias_of": "real"},
"twin-latest": {"alias_of": "real"},
"ghost-latest": {"alias_of": "ghost"},
"chain-latest": {"alias_of": "real-latest"},
"listed-latest": {"alias_of": "missing", "aliases": ["extra"]},
}
with caplog.at_level("WARNING"):
finalized: Final = _finalize_model_cost_map(raw)
assert finalized["real-latest"] == finalized["real"]
assert "alias_of" not in finalized["real"]
assert set(finalized) == {"real", "other", "real-latest", "twin-latest"}
assert finalized["real-latest"]["litellm_provider"] == "openai"
assert finalized["twin-latest"] == finalized["other"]
assert {"ghost-latest", "chain-latest", "listed-latest", "twin-latest"} == {
record.getMessage().split("'")[1] for record in caplog.records
}
def test_finalize_with_no_block_clears_rules():
previous = list(get_fallback_generalization_rules())
try:
@ -806,6 +830,7 @@ def _vcr_outcome_gate(request, vcr):
yield
record_vcr_outcome(request, vcr)
@pytest.fixture(scope="session")
def event_loop():
try:
@ -815,6 +840,7 @@ def event_loop():
yield loop
loop.close()
@pytest.fixture(scope="function")
def setup_and_teardown(event_loop):
import litellm
@ -848,6 +874,7 @@ def setup_and_teardown(event_loop):
if pending:
event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
_SCALAR_DEFAULTS = {
"num_retries": getattr(litellm, "num_retries", None),
"set_verbose": getattr(litellm, "set_verbose", False),
@ -862,6 +889,7 @@ _SCALAR_DEFAULTS = {
"cohere_key": getattr(litellm, "cohere_key", None),
}
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestCheckIsValidDict:
"""Unit tests for _check_is_valid_dict."""
@ -886,6 +914,7 @@ class TestCheckIsValidDict:
"""Non-empty dict should pass."""
assert GetModelCostMap._check_is_valid_dict({"model": {}}) is True
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestCheckModelCountNotReduced:
"""Unit tests for _check_model_count_not_reduced."""
@ -940,6 +969,7 @@ class TestCheckModelCountNotReduced:
is True
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestValidateModelCostMap:
"""Unit tests for validate_model_cost_map (combines both checks)."""
@ -976,6 +1006,7 @@ class TestValidateModelCostMap:
is True
)
def _fetch_with_single_outcome(monkeypatch, outcome):
from litellm.litellm_core_utils import get_model_cost_map as module
@ -1051,6 +1082,7 @@ class TestGetModelCostMapFallback:
assert isinstance(result, dict)
assert len(result) > 0
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestBackupModelCostMapExists:
"""Validates the local backup file is always present and valid."""
@ -1066,6 +1098,7 @@ class TestBackupModelCostMapExists:
backup = GetModelCostMap.load_local_model_cost_map()
assert len(backup) > 100, f"Backup has only {len(backup)} models, expected > 100"
@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown")
class TestBadHostedModelCostMap:
"""
@ -1130,12 +1163,14 @@ class TestBadHostedModelCostMap:
finally:
litellm.model_cost = original
@pytest.fixture()
def _vcr_outcome_gate_local_testing(request, vcr):
install_live_call_probe(request, vcr)
yield
record_vcr_outcome(request, vcr)
@pytest.fixture(scope="function")
def isolate_litellm_state():
"""
@ -1188,6 +1223,7 @@ def isolate_litellm_state():
setattr(litellm, attr, original_value)
_invalidate_model_cost_lowercase_map()
_SCALAR_DEFAULTS_local_testing = {
"num_retries": getattr(litellm, "num_retries", None),
"num_retries_per_request": getattr(litellm, "num_retries_per_request", None),
@ -1208,6 +1244,7 @@ _SCALAR_DEFAULTS_local_testing = {
"api_key": getattr(litellm, "api_key", None),
}
@pytest.fixture(scope="module")
def setup_and_teardown_local_testing():
"""
@ -1230,6 +1267,7 @@ def setup_and_teardown_local_testing():
litellm.in_memory_llm_clients_cache.flush_cache()
yield
@pytest.mark.usefixtures(
"_vcr_outcome_gate_local_testing",
"isolate_litellm_state",
@ -1241,6 +1279,7 @@ def test_get_model_cost_map():
except Exception as e:
pytest.fail(f"An exception occurred: {e}")
@pytest.mark.usefixtures(
"_vcr_outcome_gate_local_testing",
"isolate_litellm_state",

69
tests/unit/litellm_core_utils/test_model_latest_aliases.py
View file

@ -3,14 +3,17 @@ from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
from litellm.litellm_core_utils.model_latest_aliases import (
expand_model_latest_alias,
get_model_latest_aliases,
is_model_latest_alias,
resolve_model_latest_alias,
)
@pytest.mark.parametrize("model", ("", "gpt-luna-latest", "openai/unknown", "gemini/gemini-pro-latest"))
@pytest.mark.parametrize("model", ("", "kimi-latest", "openai/unknown", "gemini/gemini-pro-latest"))
def test_non_alias_expansion_keeps_the_requested_name(model: str) -> None:
assert expand_model_latest_alias(model) == (model,)
@ -21,15 +24,56 @@ def test_alias_expansion_keeps_requested_and_target_names() -> None:
assert expand_model_latest_alias(f"{provider}/{alias}") == (f"{provider}/{alias}", f"{provider}/{target}")
def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models() -> None:
for provider, aliases in get_model_latest_aliases().items():
for alias, target in aliases.items():
def test_bare_alias_expands_to_its_cost_map_owner() -> None:
alias, target = next(iter(get_model_latest_aliases()["openai"].items()))
assert expand_model_latest_alias(alias) == (alias, target)
def test_latest_aliases_price_and_route_like_their_targets() -> None:
aliases: Final = get_model_latest_aliases()
assert aliases
for provider, provider_aliases in aliases.items():
for alias, target in provider_aliases.items():
prefixed: Final = f"{provider}/{alias}"
alias_key: Final = prefixed if prefixed in litellm.model_cost else alias
target_key: Final = f"{provider}/{target}" if f"{provider}/{target}" in litellm.model_cost else target
entry: Final = litellm.model_cost[target_key]
assert (entry["litellm_provider"], entry["mode"]) == (provider, "chat")
assert f"{provider}/{alias}" not in litellm.model_cost
assert litellm.model_cost.get(alias, {}).get("litellm_provider") != provider
assert litellm.get_llm_provider(f"{provider}/{alias}")[:2] == (target, provider)
entry: Final = litellm.model_cost[alias_key]
assert entry["mode"] in ("chat", "responses")
assert entry == litellm.model_cost[target_key]
assert resolve_model_latest_alias(alias, provider) == target
if provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO:
continue
assert litellm.get_llm_provider(prefixed)[:2] == (target, provider)
assert litellm.get_llm_provider(alias, custom_llm_provider=provider)[:2] == (target, provider)
@pytest.mark.parametrize(
("model", "provider", "target_provider"),
(
("bedrock/claude-haiku-latest", "bedrock", "bedrock_converse"),
("vertex_ai/claude-haiku-latest", "vertex_ai", "vertex_ai-anthropic_models"),
("azure/gpt-mini-latest", "azure", "azure"),
("fireworks_ai/kimi-latest", "fireworks_ai", "fireworks_ai"),
),
)
def test_hosted_alias_is_sent_as_the_hosts_model_id(model: str, provider: str, target_provider: str) -> None:
entry: Final = litellm.model_cost[model]
assert isinstance(entry, AliasedCostEntry)
name: Final = model.partition("/")[2]
target: Final = entry.names[0].removeprefix(f"{provider}/")
assert target != name
assert entry["litellm_provider"] == target_provider
assert litellm.get_llm_provider(model)[:2] == (target, provider)
assert is_model_latest_alias(model)
assert resolve_model_latest_alias(name, provider) == target
assert get_model_latest_aliases()[provider][name] == target
def test_alias_resolution_is_scoped_to_the_owning_provider() -> None:
alias, target = next(iter(get_model_latest_aliases()["openai"].items()))
assert resolve_model_latest_alias(alias, "openai") == target
assert resolve_model_latest_alias(alias, "anthropic") == alias
assert resolve_model_latest_alias(target, "openai") == target
@pytest.mark.parametrize(
@ -37,11 +81,14 @@ def test_latest_alias_targets_have_chat_pricing_without_shadowing_native_models(
(
("openai/gpt-luna-latest", None, True),
("gpt-luna-latest", "openai", True),
("gpt-luna-latest", None, False),
("gpt-luna-latest", None, True),
("gpt-luna-latest", "anthropic", False),
("moonshot/kimi-latest", None, True),
("kimi-latest", None, False),
("gemini/gemini-pro-latest", None, False),
("", None, False),
("openai/unknown-model", None, False),
),
)
def test_latest_alias_detection_requires_an_explicit_provider(model: str, provider: str | None, expected: bool) -> None:
def test_latest_alias_detection_follows_the_cost_map_owner(model: str, provider: str | None, expected: bool) -> None:
assert is_model_latest_alias(model, provider) is expected

31
tests/unit/litellm_core_utils/test_streaming_handler.py
View file

@ -276,6 +276,37 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp
)
def test_is_chunk_non_empty_keeps_thinking_blocks_without_other_content(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):
"""Encrypted reasoning arrives as thinking_blocks and nothing else, so that delta has to be yielded."""
chunk: Final = ModelResponseStream(
id="chunk-thinking",
object="chat.completion.chunk",
created=1,
model="chatgpt/gpt-6.1-sol",
choices=[
StreamingChoices(
index=0,
finish_reason=None,
delta=Delta(
thinking_blocks=[
{"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"}
]
),
)
],
)
assert (
initialized_custom_stream_wrapper.is_chunk_non_empty(
completion_obj={},
model_response=chunk,
response_obj={},
)
is True
)
def test_is_chunk_non_empty_with_annotations(
initialized_custom_stream_wrapper: CustomStreamWrapper,
):

717
tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py
View file

@ -2,7 +2,8 @@ import base64
import functools
import json
import logging
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from contextlib import nullcontext
from types import MappingProxyType
from typing import Any, Final, cast
@ -11,6 +12,7 @@ import pytest
import respx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_PLACEHOLDER,
)
@ -34,6 +36,7 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.anthropic import (
AnthopicMessagesAssistantMessageParam,
AnthropicMessagesRequest,
AnthropicMessagesUserMessageParam,
)
from litellm.types.llms.openai import ChatCompletionAssistantToolCall
@ -616,6 +619,218 @@ def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning
assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"]
def test_chatgpt_messages_keep_encrypted_reasoning_and_replay_it():
"""chatgpt /v1/messages ends in the Responses API, so the signed block has to survive into the next request."""
from litellm.completion_extras.litellm_responses_transformation.transformation import (
LiteLLMResponsesTransformationHandler,
)
summary_encrypted: Final = "gAAAA_summary"
bytes_encrypted: Final = "gAAAA_bytes"
messages: Final = [
AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"),
AnthopicMessagesAssistantMessageParam(
role="assistant",
content=[
{
"type": "thinking",
"thinking": "Oslo is colder.",
"signature": encrypted_reasoning_signature(summary_encrypted),
},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(bytes_encrypted)},
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
],
),
AnthropicMessagesUserMessageParam(
role="user",
content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}],
),
]
kept: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=messages,
model="chatgpt/gpt-6.1-sol",
custom_llm_provider="chatgpt",
)
assistant: Final = kept[1]
assert [block["type"] for block in assistant["thinking_blocks"]] == ["thinking", "redacted_thinking"]
assert assistant["thinking_blocks"][0]["signature"] == encrypted_reasoning_signature(summary_encrypted)
assert assistant["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted)
responses_input, _instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
kept
)
reasoning: Final = tuple(item for item in responses_input if item.get("type") == "reasoning")
assert [item["encrypted_content"] for item in reasoning] == [summary_encrypted, bytes_encrypted]
types: Final = [item.get("type") for item in responses_input]
assert types.index("reasoning") < types.index("function_call")
stripped: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=messages,
model="gpt-4o",
custom_llm_provider="openai",
)
assert not stripped[1].get("thinking_blocks")
prefixed: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
messages=messages,
model="responses/gpt-5-mini",
custom_llm_provider="openai",
)
assert prefixed[1]["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted)
def test_translate_openai_reasoning_items_become_signed_thinking_blocks():
encrypted: Final = "gAAAA_outbound"
openai_choices: Final = [
Choices(
message=Message(
role="assistant",
content="Oslo.",
reasoning_items=[
{
"type": "reasoning",
"id": "rs_1",
"encrypted_content": encrypted,
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
}
],
tool_calls=[
ChatCompletionAssistantToolCall(
id="call_1",
type="function",
function=Function(name="get_weather", arguments='{"city": "Oslo"}'),
)
],
)
)
]
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
assert result[0]["type"] == "thinking"
assert result[0]["thinking"] == "Oslo is colder."
assert result[0]["signature"] == encrypted_reasoning_signature(encrypted)
assert result[1]["type"] == "text"
assert result[2]["type"] == "tool_use"
def test_translate_openai_reasoning_items_without_summary_become_redacted_thinking():
encrypted: Final = "gAAAA_bytes_only"
openai_choices: Final = [
Choices(
message=Message(
role="assistant",
content=None,
reasoning_items=[
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
],
)
)
]
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
assert result == [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(encrypted)},
]
def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_blocks_already_carry_them():
encrypted: Final = "gAAAA_once"
signature: Final = encrypted_reasoning_signature(encrypted)
openai_choices: Final = [
Choices(
message=Message(
role="assistant",
content="Oslo.",
thinking_blocks=[{"type": "redacted_thinking", "data": signature}],
reasoning_items=[
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
],
)
)
]
result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices)
assert [block["type"] for block in result] == ["redacted_thinking", "text"]
def _replayed_reasoning_request(tools: list[dict[str, object]]) -> AnthropicMessagesRequest:
return AnthropicMessagesRequest(
model="azure/gpt-5.4-nano",
max_tokens=64,
messages=[
AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"),
AnthopicMessagesAssistantMessageParam(
role="assistant",
content=[
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_azure_tools")},
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
],
),
AnthropicMessagesUserMessageParam(
role="user",
content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}],
),
],
tools=tools,
)
@pytest.mark.parametrize(
("tools", "kept"),
[
pytest.param(
[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
True,
id="function_tools_bridge_to_responses",
),
pytest.param([], False, id="no_tools_stay_on_chat"),
],
)
def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_responses(
tools: list[dict[str, object]], kept: bool
) -> None:
openai_request, _mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
_replayed_reasoning_request(tools),
custom_llm_provider="azure",
api_base="https://keep-check.openai.azure.com",
)
assistant: Final = openai_request["messages"][1]
assert bool(assistant.get("thinking_blocks")) is kept
@pytest.mark.parametrize(
("deployment_params", "kept"),
[
pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"),
pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"),
pytest.param({}, False, id="no_summary_stays_on_chat"),
],
)
def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses(
deployment_params: dict[str, object], kept: bool
) -> None:
from litellm.llms.anthropic.messages.adapters.handler import (
LiteLLMMessagesToCompletionTransformationHandler,
)
request: Final = _replayed_reasoning_request([])
completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
max_tokens=64,
messages=cast(list[dict[str, object]], request["messages"]),
model="azure/gpt-5.1",
thinking={"type": "enabled", "budget_tokens": 2048},
extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params},
)
assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1]
assert bool(assistant.get("thinking_blocks")) is kept
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.
@ -5578,3 +5793,503 @@ async def test_chat_bridge_tool_result_text_block_without_text_reaches_the_model
],
}
]
_RETRY_API_BASE: Final = "https://encrypted-retry.openai.azure.com"
_RETRY_URL: Final = r".*encrypted-retry\.openai\.azure\.com/openai/.*responses.*"
_RETRY_TOOLS: Final = [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}]
_RETRY_REJECTION: Final = httpx.Response(
400,
json={
"error": {
"message": "The encrypted content gAAA_foreign could not be verified.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_encrypted_content",
}
},
)
_UNRELATED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}}
)
_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}}
)
_RETRY_REPLY: Final = {
"id": "resp_retry",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.4-nano",
"output": [
{
"type": "message",
"id": "msg_retry",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Oslo", "annotations": []}],
}
],
"usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12},
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
}
def _history_with_foreign_reasoning() -> list[dict[str, object]]:
return [
{"role": "user", "content": "Which city is colder?"},
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}]},
]
def _sent_reasoning_items(request: httpx.Request) -> list[object]:
sent: Final = json.loads(request.read())
return [item for item in sent["input"] if item.get("type") == "reasoning"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter,
caplog: pytest.LogCaptureFixture,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
response: Final = await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
warnings: Final = tuple(
record.getMessage()
for record in caplog.records
if record.levelno >= logging.WARNING and record.name == "LiteLLM"
)
retry_warnings: Final = tuple(message for message in warnings if "retrying once without it" in message)
assert route.call_count == 2
assert len(_sent_reasoning_items(route.calls[0].request)) == 1
assert _sent_reasoning_items(route.calls[1].request) == []
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
assert retry_warnings == (
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content",
)
assert all(
"gAAA_foreign" not in message and "litellm_encrypted_reasoning:" not in message for message in warnings
)
@pytest.mark.usefixtures("_chat_bridge_httpx")
def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
response: Final = litellm.anthropic.messages.create(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert route.call_count == 2
assert _sent_reasoning_items(route.calls[1].request) == []
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_async_messages_with_num_retries_send_the_rejected_reasoning_only_once(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
num_retries=2,
)
assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0]
@pytest.mark.usefixtures("_chat_bridge_httpx")
def test_sync_messages_with_global_num_retries_send_the_rejected_reasoning_once_and_keep_the_setting(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "num_retries", 2)
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
litellm.anthropic.messages.create(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 0]
assert litellm.num_retries == 2
_SERVER_ERROR: Final = httpx.Response(
500, json={"error": {"message": "The server had an error while processing your request.", "type": "server_error"}}
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_async_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
response: Final = await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
num_retries=2,
)
assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0]
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
@pytest.mark.usefixtures("_chat_bridge_httpx")
def test_sync_messages_retry_a_server_error_with_the_reasoning_and_drop_it_only_once_rejected(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_SERVER_ERROR, _RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
response: Final = litellm.anthropic.messages.create(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
num_retries=2,
)
assert [len(_sent_reasoning_items(call.request)) for call in route.calls] == [1, 1, 0]
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
class _FailureLog(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.rejections: tuple[str, ...] = ()
def _record(self, kwargs: Mapping[str, object]) -> None:
error: Final = str(kwargs.get("exception"))
self.rejections = (
*self.rejections,
next((phrase for phrase in _UPSTREAM_REJECTION_PHRASES if phrase in error), error),
)
def log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs)
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs)
_UPSTREAM_REJECTION_PHRASES: Final = ("could not be verified", "max_output_tokens is too small")
_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = (
pytest.param(httpx.Response(200, json=_RETRY_REPLY), (), id="served"),
pytest.param(_UNRELATED_REJECTION, ("max_output_tokens is too small",), id="rejected"),
)
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN)
async def test_async_messages_log_only_the_outcome_of_the_turn_sent_again(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sent_again: httpx.Response,
logged_rejections: tuple[str, ...],
) -> None:
failure_log: Final = _FailureLog()
monkeypatch.setattr(litellm, "callbacks", [failure_log])
respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again])
with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext():
await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert failure_log.rejections == logged_rejections
@pytest.mark.usefixtures("_chat_bridge_httpx")
@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN)
def test_sync_messages_log_only_the_outcome_of_the_turn_sent_again(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sent_again: httpx.Response,
logged_rejections: tuple[str, ...],
) -> None:
failure_log: Final = _FailureLog()
monkeypatch.setattr(litellm, "callbacks", [failure_log])
respx_mock.post(url__regex=_RETRY_URL).mock(side_effect=[_RETRY_REJECTION, sent_again])
with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext():
litellm.anthropic.messages.create(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert failure_log.rejections == logged_rejections
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
@pytest.mark.parametrize(
("messages", "rejection"),
[
pytest.param(
[{"role": "user", "content": "Which city is colder?"}],
_RETRY_REJECTION,
id="no_encrypted_reasoning_to_drop",
),
pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"),
pytest.param(
_history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection"
),
],
)
async def test_messages_do_not_retry_other_rejections(
respx_mock: respx.MockRouter, messages: list[dict[str, object]], rejection: httpx.Response
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(return_value=rejection)
with pytest.raises(litellm.BadRequestError):
await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=messages,
)
assert route.call_count == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_messages_do_not_retry_when_the_translated_request_already_dropped_the_reasoning(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_CHAT_BRIDGE_URL).mock(return_value=_RETRY_REJECTION)
with pytest.raises(litellm.BadRequestError):
await litellm.anthropic.messages.acreate(
model=_CHAT_BRIDGE_MODEL,
api_base=_CHAT_BRIDGE_API_BASE,
api_key="fake-key",
max_tokens=64,
messages=_history_with_foreign_reasoning(),
)
assert route.call_count == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_openai_thinking_on_the_chat_adapter_replays_the_reasoning_its_responses_route_minted(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", True)
route: Final = respx_mock.post(url__regex=r".*openai-thinking\.example/v1/responses").mock(
return_value=httpx.Response(200, json=_RETRY_REPLY)
)
await litellm.anthropic.messages.acreate(
model="openai/gpt-5.4-nano",
api_base="https://openai-thinking.example/v1",
api_key="fake-key",
max_tokens=2048,
thinking={"type": "enabled", "budget_tokens": 1024},
messages=[
{"role": "user", "content": "Which city is colder?"},
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")},
{"type": "text", "text": "Oslo."},
],
},
{"role": "user", "content": "And by how much?"},
],
)
assert route.call_count == 1
assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"]
def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes:
return b"".join(
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()
for number, event in enumerate(events)
)
def _reasoning_item(item_id: str, encrypted: str, summary: str = "") -> dict[str, object]:
return {
"type": "reasoning",
"id": item_id,
"summary": [{"type": "summary_text", "text": summary}] if summary else [],
"encrypted_content": encrypted,
}
def _streamed_blocks(frames: str) -> list[tuple[str, str, str | None]]:
events: Final = tuple(
json.loads(line.removeprefix("data: ")) for line in frames.splitlines() if line.startswith("data: ")
)
deltas: Final = tuple(event for event in events if event["type"] == "content_block_delta")
def block_deltas(index: int) -> tuple[Mapping[str, str], ...]:
return tuple(delta["delta"] for delta in deltas if delta["index"] == index)
return [
(
event["content_block"]["type"],
"".join(delta.get("thinking", "") for delta in block_deltas(event["index"])),
next(
(delta["signature"] for delta in block_deltas(event["index"]) if delta["type"] == "signature_delta"),
None,
),
)
for event in events
if event["type"] == "content_block_start"
]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_streamed_reasoning_summaries_stay_with_their_own_encrypted_item(
respx_mock: respx.MockRouter,
) -> None:
response: Final = {**_RETRY_REPLY, "id": "resp_two_summaries", "status": "in_progress", "output": []}
message: Final = {
"type": "message",
"id": "msg_two_summaries",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Oslo", "annotations": []}],
}
first: Final = _reasoning_item("rs_1", "gAAA_first", "Oslo is further north.")
second: Final = _reasoning_item("rs_2", "gAAA_second", "Check the forecast.")
completed: Final = {**response, "status": "completed", "output": [first, second, message]}
def summary_delta(output_index: int, item_id: str, text: str) -> dict[str, object]:
return {
"type": "response.reasoning_summary_text.delta",
"output_index": output_index,
"item_id": item_id,
"summary_index": 0,
"delta": text,
}
events: Final = (
{"type": "response.created", "response": response},
{"type": "response.output_item.added", "output_index": 0, "item": _reasoning_item("rs_1", "")},
summary_delta(0, "rs_1", "Oslo is further north."),
{"type": "response.output_item.done", "output_index": 0, "item": first},
{"type": "response.output_item.added", "output_index": 1, "item": _reasoning_item("rs_2", "")},
summary_delta(1, "rs_2", "Check the forecast."),
{"type": "response.output_item.done", "output_index": 1, "item": second},
{"type": "response.output_item.added", "output_index": 2, "item": {**message, "content": []}},
{
"type": "response.output_text.delta",
"output_index": 2,
"item_id": "msg_two_summaries",
"content_index": 0,
"delta": "Oslo",
},
{"type": "response.output_item.done", "output_index": 2, "item": message},
{"type": "response.completed", "response": completed},
)
respx_mock.post(url__regex=_RETRY_URL).mock(
return_value=httpx.Response(200, headers={"content-type": "text/event-stream"}, content=_responses_sse(events))
)
stream: Final = await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
stream=True,
messages=[{"role": "user", "content": "Which city is colder?"}],
)
frames: Final = b"".join([chunk async for chunk in stream]).decode()
assert _streamed_blocks(frames) == [
("thinking", "Oslo is further north.", encrypted_reasoning_signature("gAAA_first")),
("thinking", "Check the forecast.", encrypted_reasoning_signature("gAAA_second")),
("text", "", None),
]

14
tests/unit/llms/anthropic/messages/native/test_anthropic_messages_transformation.py
View file

@ -721,3 +721,17 @@ def test_a_first_party_messages_request_never_fetches_the_remote_beta_headers_co
assert request["messages"][1] == {"role": "system", "content": "be terse"}
assert not remote_config.called
@pytest.mark.parametrize(
"model, expected",
[
("claude-sonnet-5", {"type": "disabled"}),
("claude-haiku-4-5", None),
("claude-sonnet-5-5", None),
],
)
def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None:
params: Final[dict[str, object]] = {"reasoning_effort": "none", "output_config": {"effort": "high"}}
AnthropicMessagesConfig._translate_reasoning_effort_to_anthropic(model, params, 1024, "anthropic")
assert params == ({"thinking": expected} if expected is not None else {})

292
tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py
View file

@ -3,20 +3,24 @@ import json
import os
import sys
from collections.abc import AsyncIterator, Mapping
from contextlib import nullcontext
from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.messages.responses_adapters.handler import (
LiteLLMMessagesToResponsesAPIHandler,
_build_responses_kwargs,
@ -1135,3 +1139,291 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
]
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
REJECTED_REPLAYED_REASONING: Final = {
"error": {
"message": "The encrypted content gAAA_foreign could not be verified.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_encrypted_content",
}
}
UNVERIFIED_ORGANIZATION: Final = {
"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}
}
def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]:
reasoning: Final = (
({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},)
if replayed_reasoning
else ()
)
return [
{"role": "user", "content": "List the logs."},
{"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]},
]
def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]:
return [
sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"])
for call in upstream.calls
]
def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool):
return _upstream_after(respx_mock, monkeypatch, (first,), stream)
def _upstream_after(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, failures: tuple[httpx.Response, ...], stream: bool
) -> respx.Route:
monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
served: Final = (
httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
if stream
else httpx.Response(200, json=TEXT_RESPONSE)
)
return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[*failures, served])
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream)
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=stream,
custom_llm_provider="openai",
)
events: Final = (
[json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)]
if stream
else [cast(Mapping[str, object], response)]
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert events[-1]["type"] in ("message", "message_stop")
def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False)
response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}]
@pytest.mark.asyncio
async def test_openai_turn_with_num_retries_sends_the_rejected_reasoning_only_once(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False)
await litellm.anthropic.messages.acreate(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
num_retries=2,
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
def test_sync_openai_turn_with_global_num_retries_sends_the_rejected_reasoning_once_and_keeps_the_setting(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "num_retries", 2)
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False)
litellm.anthropic.messages.create(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert litellm.num_retries == 2
_SERVER_ERROR: Final = {
"error": {"message": "The server had an error while processing your request.", "type": "server_error"}
}
@pytest.mark.asyncio
async def test_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream_after(
respx_mock,
monkeypatch,
(httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)),
False,
)
response: Final = await litellm.anthropic.messages.acreate(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
num_retries=2,
)
assert _replayed_reasoning_counts(upstream) == [1, 1, 0]
assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}]
def test_sync_openai_turn_retries_a_server_error_with_the_reasoning_and_drops_it_only_once_rejected(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream_after(
respx_mock,
monkeypatch,
(httpx.Response(500, json=_SERVER_ERROR), httpx.Response(400, json=REJECTED_REPLAYED_REASONING)),
False,
)
response: Final = litellm.anthropic.messages.create(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
num_retries=2,
)
assert _replayed_reasoning_counts(upstream) == [1, 1, 0]
assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}]
class _FailureLog(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.rejections: tuple[str, ...] = ()
def _record(self, kwargs: Mapping[str, object]) -> None:
error: Final = str(kwargs.get("exception"))
self.rejections = (
*self.rejections,
next((phrase for phrase in ("encrypted content", "organization") if phrase in error), error),
)
def log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs)
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs)
_OUTCOMES_OF_THE_TURN_SENT_AGAIN: Final = (
pytest.param(httpx.Response(200, json=TEXT_RESPONSE), (), id="served"),
pytest.param(httpx.Response(400, json=UNVERIFIED_ORGANIZATION), ("organization",), id="rejected"),
)
def _rejected_then(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sent_again: httpx.Response) -> None:
monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
respx_mock.post("https://api.openai.com/v1/responses").mock(
side_effect=[httpx.Response(400, json=REJECTED_REPLAYED_REASONING), sent_again]
)
@pytest.mark.asyncio
@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN)
async def test_openai_turn_logs_only_the_outcome_of_the_turn_sent_again(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sent_again: httpx.Response,
logged_rejections: tuple[str, ...],
) -> None:
failure_log: Final = _FailureLog()
monkeypatch.setattr(litellm, "callbacks", [failure_log])
_rejected_then(respx_mock, monkeypatch, sent_again)
with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext():
await litellm.anthropic.messages.acreate(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
)
assert failure_log.rejections == logged_rejections
@pytest.mark.parametrize(("sent_again", "logged_rejections"), _OUTCOMES_OF_THE_TURN_SENT_AGAIN)
def test_sync_openai_turn_logs_only_the_outcome_of_the_turn_sent_again(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
sent_again: httpx.Response,
logged_rejections: tuple[str, ...],
) -> None:
failure_log: Final = _FailureLog()
monkeypatch.setattr(litellm, "callbacks", [failure_log])
_rejected_then(respx_mock, monkeypatch, sent_again)
with pytest.raises(litellm.BadRequestError) if logged_rejections else nullcontext():
litellm.anthropic.messages.create(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
)
assert failure_log.rejections == logged_rejections
@pytest.mark.asyncio
@pytest.mark.parametrize(
"replayed_reasoning, rejection",
[
pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"),
pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"),
],
)
async def test_openai_turn_is_not_sent_again_for_other_rejections(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
replayed_reasoning: bool,
rejection: Mapping[str, object],
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False)
with pytest.raises(litellm.BadRequestError):
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=replayed_reasoning),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert upstream.call_count == 1

31
tests/unit/llms/anthropic/test_anthropic_reasoning_effort.py
View file

@ -5,11 +5,42 @@ Verifies that reasoning_effort=None returns None for all models,
including Claude Opus 4.6.
"""
from typing import Final
import pytest
from litellm.exceptions import BadRequestError
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
@pytest.mark.parametrize(
"model, expected",
[
("claude-sonnet-5", {"type": "disabled"}),
("claude-opus-5", {"type": "disabled"}),
("claude-haiku-5-5", {"type": "disabled"}),
("claude-haiku-4-5", None),
("claude-sonnet-5-5", None),
("claude-opus-5-5", None),
("claude-fable-5", None),
("claude-mythos-5", None),
],
)
def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None:
mapped: Final = AnthropicConfig().map_openai_params({"reasoning_effort": "none"}, {}, model, False)
assert mapped.get("thinking") == expected
assert "output_config" not in mapped
@pytest.mark.parametrize("effort", [None, "bogus"])
def test_default_thinking_preserves_absent_and_invalid_effort(effort: str | None) -> None:
if effort is None:
assert AnthropicConfig.map_reasoning_effort(effort, "claude-sonnet-5", "anthropic") is None
else:
with pytest.raises(BadRequestError, match="Invalid reasoning_effort"):
AnthropicConfig().map_openai_params({"reasoning_effort": effort}, {}, "claude-sonnet-5", False)
class TestMapReasoningEffort:
def test_none_returns_none_for_opus_4_6(self):
"""reasoning_effort=None should return None for Opus 4.6, not adaptive."""

15
tests/unit/llms/bedrock/chat/test_converse_transformation.py
View file

@ -12203,6 +12203,21 @@ def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hin
assert messages == messages_before
@pytest.mark.parametrize(
"model, expected",
[
("us.anthropic.claude-sonnet-5", {"type": "disabled"}),
("us.anthropic.claude-haiku-4-5", None),
("us.anthropic.claude-sonnet-5-5", None),
],
)
def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None:
params: Final[dict[str, object]] = {}
AmazonConverseConfig()._handle_reasoning_effort_parameter(model, "none", params)
assert params.get("thinking") == expected
assert "output_config" not in params
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
) -> None:

15
tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py
View file

@ -3220,3 +3220,18 @@ def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_co
assert "cache_control" not in json.dumps(body["messages"][0])
assert json.dumps(body["messages"]).count('"cache_control"') == 3
@pytest.mark.parametrize(
"model, expected",
[
("databricks-claude-sonnet-5", {"type": "disabled"}),
("databricks-claude-haiku-4-5", None),
("databricks-claude-sonnet-5-5", None),
],
)
def test_none_disables_only_optional_default_thinking(model: str, expected: dict[str, str] | None) -> None:
params: Final = _map_reasoning_effort(model, "none")
assert params.get("thinking") == expected
assert "output_config" not in params
assert "reasoning_effort" not in params

2
tests/unit/llms/databricks/test_databricks_cost_calculator.py
View file

@ -6,6 +6,7 @@ from typing import Final
import pytest
import litellm
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
from litellm.llms.databricks.cost_calculator import cost_per_token
from litellm.types.utils import ModelInfo, Usage
@ -213,6 +214,7 @@ def test_every_model_without_published_cache_dbu_bills_cache_writes_at_its_own_i
if model.startswith("databricks/")
and info.get("input_cost_per_token")
and model not in PUBLISHED_DBU_PER_MILLION
and not (isinstance(info, AliasedCostEntry) and info.names[0] != model)
]
for model in without_published_rates:

5
tests/unit/llms/moonshot/test_moonshot_chat_transformation.py
View file

@ -13,7 +13,7 @@ import pytest
import litellm
import litellm.utils
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap, get_model_cost_map
from litellm.litellm_core_utils.model_latest_aliases import get_model_latest_aliases
from litellm.llms.moonshot.chat.transformation import MoonshotChatConfig
@ -715,7 +715,8 @@ class TestMoonshotReasoningEffort:
@pytest.fixture(autouse=True)
def force_local_model_cost(self, monkeypatch):
monkeypatch.setattr(litellm, "model_cost", GetModelCostMap.load_local_model_cost_map())
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
@pytest.mark.parametrize("model", ["kimi-k3", "kimi-k2.5", "kimi-k2.6"])
def test_reasoning_model_supports_reasoning_effort(self, model):

5
tests/unit/llms/vertex_ai/image_edit/test_vertex_ai_image_edit_transformation.py
View file

@ -231,10 +231,10 @@ class TestVertexAIGeminiImageEditTransformation:
)
def test_transform_image_edit_request_without_image_raises(self) -> None:
"""Test that missing image raises ValueError"""
"""Test that missing image raises BadRequestError with 400 status code"""
optional_params = {}
with pytest.raises(ValueError, match="requires at least one image"):
with pytest.raises(litellm.BadRequestError, match="requires at least one image") as exc_info:
self.config.transform_image_edit_request(
model=self.model,
prompt=self.prompt,
@ -243,6 +243,7 @@ class TestVertexAIGeminiImageEditTransformation:
litellm_params=MagicMock(),
headers={},
)
assert exc_info.value.status_code == 400
def test_validate_environment_with_litellm_params(self) -> None:
"""Test validate_environment uses credentials from litellm_params"""

30
tests/unit/proxy/common_utils/test_encrypt_decrypt_utils.py
View file

@ -6,8 +6,11 @@ gate, and the backward-compatibility guarantees that let legacy XSalsa20-Poly130
(nacl) ciphertext and new AES values coexist and decrypt correctly.
"""
import _hashlib
import base64
import hmac
import re
from typing import Final
import pytest
@ -17,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_bearer_token,
decrypt_if_encrypted_with,
decrypt_value_helper,
derive_key_from_salt_key,
encrypt_bearer_token,
encrypt_value,
encrypt_value_helper,
@ -266,3 +270,29 @@ def test_bearer_token_uses_only_header_safe_characters(length: int):
assert re.fullmatch(r"kind_a_[A-Za-z0-9_-]+", token), token
assert decrypt_bearer_token(token, prefix="kind_a_") == "x" * length
_needs_shared_openssl: Final = pytest.mark.skipif(
not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach"
)
@_needs_shared_openssl
@pytest.mark.usefixtures("fips_openssl")
@pytest.mark.parametrize("salt_key", ["sk-" + bytes(range(32)).hex(), bytes(range(32)).hex().upper()])
def test_derive_key_from_salt_key_is_sp800_108_counter_mode_hmac_sha256(
monkeypatch: pytest.MonkeyPatch, salt_key: str
) -> None:
label: Final = b"litellm-test:label:"
context: Final = b"sk-test-master-key"
monkeypatch.setenv("LITELLM_SALT_KEY", salt_key)
fixed_input: Final = (1).to_bytes(4, "big") + label + b"\x00" + context + (256).to_bytes(4, "big")
assert derive_key_from_salt_key(label, context) == hmac.digest(bytes(range(32)), fixed_input, "sha256").hex()
@_needs_shared_openssl
@pytest.mark.skipif(_hashlib.get_fips_mode() == 1, reason="this OpenSSL runs a FIPS provider")
def test_derive_key_from_salt_key_refuses_an_openssl_without_a_fips_provider(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + bytes(range(32)).hex())
with pytest.raises(ValueError, match="OpenSSL could not run KBKDF"):
derive_key_from_salt_key(b"litellm-test:label:", b"sk-test-master-key")

1
tests/unit/proxy/conftest.py
View file

@ -523,6 +523,7 @@ def postgres_span_names() -> Iterator[Callable[[], Awaitable[tuple[str, ...]]]]:
def fips_openssl(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(_hashlib, "get_fips_mode", lambda: 1)
monkeypatch.setattr(_hashlib, "scrypt", MagicMock(side_effect=ValueError("[digital envelope routines] unsupported")))
monkeypatch.setattr("litellm.proxy.common_utils.encrypt_decrypt_utils._KBKDF_PROPERTIES", None)
@pytest.fixture

142
tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
View file

@ -247,6 +247,88 @@ class TestContentFilterGuardrail:
assert "[EMAIL_REDACTED]" in result[0]
assert "test@example.com" not in result[0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"pattern_names",
[("aws_secret_key", "openai_api_key"), ("openai_api_key", "aws_secret_key")],
)
async def test_apply_guardrail_masks_whole_key_when_another_pattern_matches_inside_it(
self, pattern_names: tuple[str, str]
):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-overlapping-patterns",
patterns=[
ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK)
for name in pattern_names
],
)
openai_key: Final = "sk-proj-" + "Ab3" * 13 + "x-" + "Cd4" * 11 + "T3BlbkFJ" + "Ef5" * 24 + "gh"
aws_secret_key: Final = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
guardrailed_inputs: Final = await guardrail.apply_guardrail(
inputs={"texts": [f"key {openai_key} and secret {aws_secret_key}"]},
request_data={},
input_type="request",
)
assert guardrailed_inputs.get("texts") == [
"key [OPENAI_API_KEY_REDACTED] and secret [AWS_SECRET_KEY_REDACTED]"
]
@pytest.mark.asyncio
async def test_apply_guardrail_does_not_block_a_match_an_earlier_mask_pattern_already_covers(self):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-mask-then-block",
patterns=[
ContentFilterPattern(
pattern_type="prebuilt", pattern_name="generic_api_key", action=ContentFilterAction.MASK
),
ContentFilterPattern(
pattern_type="prebuilt", pattern_name="aws_secret_key", action=ContentFilterAction.BLOCK
),
],
)
guardrailed_inputs: Final = await guardrail.apply_guardrail(
inputs={"texts": ["config api_key=" + "A1b2C3d4E5" * 4]},
request_data={},
input_type="request",
)
assert guardrailed_inputs.get("texts") == ["config [GENERIC_API_KEY_REDACTED]"]
@pytest.mark.asyncio
async def test_apply_guardrail_masks_a_contextual_match_an_earlier_mask_moved_next_to_its_keyword(self):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-contextual-after-mask",
patterns=[
ContentFilterPattern(pattern_type="prebuilt", pattern_name="email", action=ContentFilterAction.MASK),
ContentFilterPattern(pattern_type="prebuilt", pattern_name="au_tfn", action=ContentFilterAction.MASK),
],
)
guardrailed_inputs: Final = await guardrail.apply_guardrail(
inputs={"texts": ["TFN jane.doe@example.com 123456782"]},
request_data={},
input_type="request",
)
assert guardrailed_inputs.get("texts") == ["TFN [EMAIL_REDACTED] [AU_TFN_REDACTED]"]
@pytest.mark.asyncio
async def test_apply_guardrail_counts_only_the_unnamed_regex_that_matched(self):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-unnamed-regex-count",
patterns=[
ContentFilterPattern(pattern_type="regex", pattern=r"Key\d+", action=ContentFilterAction.MASK),
ContentFilterPattern(pattern_type="regex", pattern=r"Token\d+", action=ContentFilterAction.MASK),
],
)
request_data: Final = {"metadata": {}}
await guardrail.apply_guardrail(
inputs={"texts": ["use Key1 here"]},
request_data=request_data,
input_type="request",
)
entry: Final = request_data["metadata"]["standard_logging_guardrail_information"][0]
assert entry["masked_entity_count"] == {"custom_regex": 1}
@pytest.mark.asyncio
async def test_apply_guardrail_blocked_word_mask(self):
"""
@ -919,12 +1001,21 @@ class TestContentFilterGuardrail:
# masked_entity_count for email is the real count, not N×.
assert entry["masked_entity_count"].get("email") == 1
@staticmethod
@classmethod
async def _collect_streamed_text(
cls,
guardrail: ContentFilterGuardrail,
chunks: list[str],
metadata: dict[str, list[StandardLoggingGuardrailInformation]],
) -> str:
return "".join(await cls._collect_streamed_deltas(guardrail, chunks, metadata))
@staticmethod
async def _collect_streamed_deltas(
guardrail: ContentFilterGuardrail,
chunks: list[str],
metadata: dict[str, list[StandardLoggingGuardrailInformation]],
) -> list[str]:
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
async def mock_stream():
@ -951,7 +1042,7 @@ class TestContentFilterGuardrail:
request_data={"messages": [], "model": "gpt-4o", "metadata": metadata},
):
yielded.append(chunk.choices[0].delta.content or "")
return "".join(yielded)
return yielded
@pytest.mark.asyncio
async def test_streaming_hook_scans_bounded_window_per_chunk(self):
@ -1068,6 +1159,53 @@ class TestContentFilterGuardrail:
assert entry["guardrail_status"] == "guardrail_intervened"
assert [d["keyword"] for d in entry["guardrail_response"]] == [phrase]
@pytest.mark.asyncio
@pytest.mark.parametrize(
("pattern_name", "key"),
[
("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74),
("openai_api_key", "sk-svcacct-" + "a" * 200 + "T3BlbkFJ" + "b" * 200),
("anthropic_api_key", "sk-ant-api03-" + "a" * 93 + "AA"),
("gemini_api_key", "AQ." + "a" * 55),
],
)
async def test_streaming_hook_masks_whole_provider_key_split_across_chunks(self, pattern_name: str, key: str):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-streaming-provider-key",
patterns=[
ContentFilterPattern(
pattern_type="prebuilt",
pattern_name=pattern_name,
action=ContentFilterAction.MASK,
)
],
event_hook=GuardrailEventHooks.post_call,
)
text: Final = f"Use {key} to call the API."
chunks: Final = [text[i : i + 4] for i in range(0, len(text), 4)]
streamed: Final = await self._collect_streamed_text(guardrail, chunks, {})
assert streamed == f"Use [{pattern_name.upper()}_REDACTED] to call the API."
@pytest.mark.asyncio
async def test_streaming_hook_keeps_default_holdback_without_provider_key_patterns(self):
guardrail: Final = ContentFilterGuardrail(
guardrail_name="test-streaming-older-credential-patterns",
patterns=[
ContentFilterPattern(pattern_type="prebuilt", pattern_name=name, action=ContentFilterAction.MASK)
for name in ("aws_access_key", "aws_secret_key", "github_token", "slack_token", "generic_api_key")
],
event_hook=GuardrailEventHooks.post_call,
)
text: Final = "plain words with nothing secret in them. " * 12
chunks: Final = [text[i : i + 10] for i in range(0, len(text), 10)]
deltas: Final = await self._collect_streamed_deltas(guardrail, chunks, {})
assert "".join(deltas) == text
assert len(next(delta for delta in reversed(deltas) if delta)) <= CONTENT_FILTER_STREAMING_HOLDBACK_CHARS
@pytest.mark.asyncio
async def test_streaming_hook_blocks_keyword_longer_than_scan_context(self):
"""

168
tests/unit/proxy/guardrails/guardrail_hooks/content_filter/test_patterns.py
View file

@ -4,6 +4,8 @@ Tests for content filter pattern loading from JSON
import json
import os
import time
from typing import Final
import pytest
@ -183,3 +185,169 @@ def test_eu_patterns_have_category():
assert (
pattern_name in eu_category_patterns
), f"Pattern {pattern_name} not in EU PII Patterns category"
def test_provider_key_patterns_are_offered_in_the_ui_as_credential_patterns() -> None:
ui_categories: Final = {pattern["name"]: pattern["category"] for pattern in get_pattern_metadata()}
assert [ui_categories.get(name) for name in ("openai_api_key", "anthropic_api_key", "gemini_api_key")] == [
"Credential Patterns"
] * 3
@pytest.mark.parametrize(
("pattern_name", "key"),
[
pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="openai-project"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-project-shortest"),
pytest.param("openai_api_key", "sk-svcacct-" + "a_b-" * 15 + "T3BlbkFJ" + "c-d_" * 15, id="openai-service-account"),
pytest.param("openai_api_key", "sk-admin-" + "a" * 58 + "T3BlbkFJ" + "b" * 58, id="openai-admin"),
pytest.param("openai_api_key", "sk-None-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-user-key"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19 + "-", id="openai-trailing-hyphen"),
pytest.param(
"anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOP", id="anthropic-versioned"
),
pytest.param(
"anthropic_api_key", "sk-ant-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTU", id="anthropic-unversioned"
),
pytest.param("anthropic_api_key", "sk-ant-" + "a" * 20, id="anthropic-shortest"),
pytest.param("anthropic_api_key", "sk-ant-api03-" + "a_b-" * 23 + "AA", id="anthropic-urlsafe"),
pytest.param(
"anthropic_api_key", "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789-", id="anthropic-trailing-hyphen"
),
pytest.param("gemini_api_key", "AIzaSyD-9tSrke72I6_AQp2h8HI4-uE6QNKQV0", id="gemini-legacy"),
pytest.param("gemini_api_key", "AIza" + "a" * 33, id="gemini-legacy-shortest"),
pytest.param("gemini_api_key", "AIza" + "a" * 36, id="gemini-legacy-longest"),
pytest.param("gemini_api_key", "AIza" + "a" * 34 + "-", id="gemini-legacy-trailing-hyphen"),
pytest.param("gemini_api_key", "AQ.Ab8RN6I-7xK9pL2mQ0vW5yZ1uX3tS4rE8qD7cB6aN5mP4oI3uY2", id="gemini-auth-key"),
pytest.param("gemini_api_key", "AQ." + "a" * 45, id="gemini-auth-key-shortest"),
pytest.param("gemini_api_key", "AQ." + "a" * 55, id="gemini-auth-key-longest"),
pytest.param("gemini_api_key", "AQ." + "a" * 49 + "-", id="gemini-auth-key-trailing-hyphen"),
],
)
def test_credential_pattern_matches_whole_key(pattern_name: str, key: str) -> None:
match: Final = get_compiled_pattern(pattern_name).search(f"I keep getting 401 with {key}, is it expired?")
assert match is not None
assert match.group(0) == key
@pytest.mark.parametrize("separator", ["\\n", "\\t", "\\u003c", "Bearer%20", "?key%3D", "?key%253D", "GOOGLE_KEY_"])
@pytest.mark.parametrize(
("pattern_name", "key"),
[
pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai"),
pytest.param("anthropic_api_key", "sk-ant-api03-" + "a" * 20, id="anthropic"),
pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"),
pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"),
],
)
def test_credential_pattern_matches_key_after_escaped_separator(pattern_name: str, key: str, separator: str) -> None:
match: Final = get_compiled_pattern(pattern_name).search(f'{{"log": "retrying with{separator}{key} failed"}}')
assert match is not None
assert match.group(0) == key
@pytest.mark.parametrize(
("pattern_name", "text"),
[
pytest.param("openai_api_key", "sk-short", id="openai-too-short"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 19, id="openai-project-too-short"),
pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 21, id="openai-legacy-too-long"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 48, id="openai-project-without-marker"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 19 + "T3BlbkFJ" + "b" * 20, id="openai-project-marker-too-early"),
pytest.param("openai_api_key", "sk-proj-" + "a" * 20 + "T3BlbkFJ" + "b" * 19, id="openai-project-marker-too-late"),
pytest.param("openai_api_key", "SK-ADMIN-dashboard-redesign-phase-two", id="openai-prefixed-slug"),
pytest.param("openai_api_key", "desk-proj-management-dashboard-v2", id="openai-inside-longer-word"),
pytest.param(
"openai_api_key", "sk-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRST", id="openai-legacy-without-marker"
),
pytest.param("openai_api_key", "sk-0123456789abcdef0123456789abcdef", id="openai-other-vendor-sk-key"),
pytest.param("openai_api_key", "sk-Xq3vR9tLm2Pa8KdW5yZgQw", id="openai-litellm-virtual-key"),
pytest.param(
"openai_api_key",
"sk-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCd-_x",
id="openai-litellm-virtual-key-raised-length",
),
pytest.param("openai_api_key", "not-a-key", id="openai-no-prefix"),
pytest.param(
"openai_api_key", "sk-ant-api03-thisisanthropicnotopenaiabc123", id="openai-ignores-anthropic-key"
),
pytest.param("openai_api_key", "ordinary sentence without any api key in it", id="openai-plain-text"),
pytest.param("anthropic_api_key", "sk-ant-" + "a" * 19, id="anthropic-too-short"),
pytest.param("anthropic_api_key", "flask-ant-design-system-components", id="anthropic-inside-longer-word"),
pytest.param(
"anthropic_api_key",
"sk-proj-abcdefghijklmnopqrstuvwxyzABCDEF1234567890",
id="anthropic-ignores-openai-project-key",
),
pytest.param(
"anthropic_api_key",
"sk-admin-abcdefghijklmnopqrstuvwxyzABCDEF123456",
id="anthropic-ignores-openai-admin-key",
),
pytest.param("anthropic_api_key", "sk-ant", id="anthropic-prefix-only"),
pytest.param("anthropic_api_key", "plain text with nothing sensitive", id="anthropic-plain-text"),
pytest.param("gemini_api_key", "AIza", id="gemini-prefix-only"),
pytest.param("gemini_api_key", "AIza" + "a" * 32, id="gemini-legacy-too-short"),
pytest.param("gemini_api_key", "AIza" + "a" * 37, id="gemini-legacy-too-long"),
pytest.param("gemini_api_key", "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB", id="gemini-wrong-prefix"),
pytest.param("gemini_api_key", "AQ.", id="gemini-auth-prefix-only"),
pytest.param("gemini_api_key", "AQ." + "a" * 44, id="gemini-auth-key-too-short"),
pytest.param("gemini_api_key", "AQ." + "a" * 56, id="gemini-auth-key-too-long"),
pytest.param("gemini_api_key", "AQ-" + "a" * 50, id="gemini-auth-key-wrong-separator"),
pytest.param("gemini_api_key", "x" + "AIza" + "a" * 35, id="gemini-inside-longer-token"),
pytest.param("gemini_api_key", "The AQ. was reported at 42 ppm today.", id="gemini-aq-acronym"),
pytest.param("gemini_api_key", "Is the question AQ. Maybe it is BQ.", id="gemini-aq-sentence-end"),
pytest.param("gemini_api_key", "plain text with no google key", id="gemini-plain-text"),
pytest.param(
"gemini_api_key",
"https://example.com/blog/aizawa-takeshi-interview-2024-tokyo-01",
id="gemini-lowercase-legacy-prefix",
),
pytest.param(
"gemini_api_key",
"ssh aq.internal-observability-metrics-collector-eu-west-01",
id="gemini-lowercase-auth-prefix",
),
],
)
def test_credential_pattern_ignores_non_keys(pattern_name: str, text: str) -> None:
assert get_compiled_pattern(pattern_name).search(text) is None
@pytest.mark.parametrize("suffix", ["请检查", "é", "ü der Schlüssel"])
@pytest.mark.parametrize(
("pattern_name", "key"),
[
pytest.param("openai_api_key", "sk-" + "a" * 20 + "T3BlbkFJ" + "b" * 20, id="openai-legacy"),
pytest.param("gemini_api_key", "AIza" + "a" * 35, id="gemini-legacy"),
pytest.param("gemini_api_key", "AQ." + "a" * 50, id="gemini-auth-key"),
],
)
def test_credential_pattern_matches_key_followed_by_non_ascii_letters(pattern_name: str, key: str, suffix: str) -> None:
match: Final = get_compiled_pattern(pattern_name).search(f"我的密钥是{key}{suffix}")
assert match is not None
assert match.group(0) == key
@pytest.mark.parametrize(
"leading",
[
pytest.param("sk-proj-" + "a" * 74 + "T3BlbkFJ" + "b" * 74, id="another-key"),
pytest.param("sk-proj-" + "x" * 20 + "T3BlbkFJ" + "y" * 192, id="key-shaped-filler"),
pytest.param("sk-proj-" + "x" * 100 + "T3BlbkFJ" + "y" * 100, id="long-key-shaped-filler"),
],
)
def test_openai_key_pattern_masks_a_key_glued_to_preceding_key_shaped_text(leading: str) -> None:
key: Final = "sk-proj-" + "c" * 74 + "T3BlbkFJ" + "d" * 74
assert get_compiled_pattern("openai_api_key").sub("[KEY]", leading + key) == "[KEY][KEY]"
@pytest.mark.parametrize("prefix", ["sk-proj-", "sk-svcacct-", "sk-admin-", "sk-None-"])
def test_openai_key_pattern_scans_repeated_prefixes_without_stalling(prefix: str) -> None:
prompt: Final = prefix * (128_000 // len(prefix))
started: Final = time.perf_counter()
match: Final = get_compiled_pattern("openai_api_key").search(prompt)
elapsed: Final = time.perf_counter() - started
assert match is None
assert elapsed < 1.0

20
tests/unit/proxy/mcp_server/outbound_credentials/conftest.py Normal file
View file

@ -0,0 +1,20 @@
from collections.abc import Iterator
import pytest
from litellm.proxy.mcp_server.outbound_credentials.bridge_credentials import envelope_keys_from_master_key
from litellm.proxy.mcp_server.outbound_credentials.session_credentials import session_keys_from_master_key
@pytest.fixture
def openssl_enforcing_fips(fips_openssl: None) -> Iterator[None]:
envelope_keys_from_master_key.cache_clear()
session_keys_from_master_key.cache_clear()
yield
envelope_keys_from_master_key.cache_clear()
session_keys_from_master_key.cache_clear()
@pytest.fixture
def fips_mode_on(openssl_enforcing_fips: None, fips_mode: None) -> None:
return None

31
tests/unit/proxy/mcp_server/outbound_credentials/test_bridge_credentials.py
View file

@ -9,6 +9,7 @@ through the consumer; and no path leaks the upstream token in a repr.
"""
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from pydantic import SecretStr
@ -318,3 +319,33 @@ def test_is_bridge_envelope_shaped_rejects_non_envelope_bearer():
assert is_bridge_envelope_shaped("Bearer sk-some-litellm-key") is False
assert is_bridge_envelope_shaped("plain-upstream-token") is False
assert is_bridge_envelope_shaped("") is False
def test_envelope_keys_without_fips_stay_the_scrypt_keys_of_the_master_key() -> None:
keys: Final = envelope_keys_from_master_key(_MASTER_KEY)
assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == (
"34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed",
"eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250",
)
@pytest.mark.usefixtures("openssl_enforcing_fips")
def test_envelope_keys_on_a_fips_openssl_without_fips_mode_stay_the_scrypt_keys(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
keys: Final = envelope_keys_from_master_key(_MASTER_KEY)
assert (keys.signing_key.get_secret_value(), keys.encryption_key.get_secret_value()) == (
"34d3bb6ae2e03cddca3417209304e9603417b89258d3979d0569880b6fbd1bed",
"eebfbee8cbe767ee136ce3a9745b44de2b2ae35d1ea46dcf72c58b811fac3250",
)
@pytest.mark.usefixtures("fips_mode_on")
@pytest.mark.parametrize("salt_key", ["", "sk-salt-change-me", _MASTER_KEY, "sk-" + "5a" * 31])
def test_envelope_keys_under_fips_refuse_a_salt_key_that_is_not_32_bytes_of_hex(
monkeypatch: pytest.MonkeyPatch, salt_key: str
) -> None:
monkeypatch.setenv("LITELLM_SALT_KEY", salt_key)
with pytest.raises(ValueError, match="LITELLM_SALT_KEY"):
envelope_keys_from_master_key(_MASTER_KEY)

63
tests/unit/proxy/mcp_server/outbound_credentials/test_session_credentials.py
View file

@ -1,6 +1,8 @@
"""Tests for the session-token KDF and the edge/token-endpoint resolvers."""
import _hashlib
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from cryptography.hazmat.primitives import serialization
@ -230,3 +232,64 @@ def test_signing_config_error_detail_never_leaks_key_material():
)
assert isinstance(resolved, SessionSigningConfigError)
assert pem.splitlines()[1] not in resolved.detail
def test_session_key_without_fips_stays_the_scrypt_key_of_the_master_key() -> None:
keys: Final = session_keys_from_master_key(MASTER_KEY)
assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d"
@pytest.mark.usefixtures("openssl_enforcing_fips")
def test_session_key_on_a_fips_openssl_without_fips_mode_stays_the_scrypt_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
keys: Final = session_keys_from_master_key(MASTER_KEY)
assert keys.signing_key.get_secret_value() == "990dce331d97862e3af47815b707c42923fc760e6740d4a77a81583a206f339d"
def _fips_keys(master_key: str) -> tuple[str, str, str]:
envelope_keys_from_master_key.cache_clear()
session_keys_from_master_key.cache_clear()
envelope: Final = envelope_keys_from_master_key(master_key)
session: Final = session_keys_from_master_key(master_key)
return (
envelope.signing_key.get_secret_value(),
envelope.encryption_key.get_secret_value(),
session.signing_key.get_secret_value(),
)
_needs_shared_openssl: Final = pytest.mark.skipif(
not hasattr(_hashlib, "__file__"), reason="this Python links OpenSSL statically, out of ctypes reach"
)
@_needs_shared_openssl
@pytest.mark.usefixtures("fips_mode_on")
def test_keys_under_fips_come_from_the_salt_key_are_distinct_and_the_same_on_every_call(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32)
first: Final = _fips_keys(MASTER_KEY)
assert first == _fips_keys(MASTER_KEY)
assert len(set(first)) == 3, first
assert first == (
"ef694d415772aef09731c47709d4d9da47f7bbfcd6484836dea09c2bb387063b",
"d8a7cd70690fbf98c5581301b2e0460f2ad714f7c74b2af1b3dac3e0aa99db85",
"5e11e470bf9c838165b70d0b2b82a7ee5e2f00d9449eeb54f7c5bf9491b64354",
)
@_needs_shared_openssl
@pytest.mark.usefixtures("fips_mode_on")
def test_keys_under_fips_all_change_when_the_master_key_rotates(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("LITELLM_SALT_KEY", "sk-" + "5a" * 32)
before: Final = _fips_keys(MASTER_KEY)
after: Final = _fips_keys("sk-a-different-master-key")
assert set(before).isdisjoint(after), (before, after)
@pytest.mark.usefixtures("fips_mode_on")
def test_session_key_under_fips_refuses_an_unset_salt_key(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
with pytest.raises(ValueError, match="LITELLM_SALT_KEY"):
session_keys_from_master_key(MASTER_KEY)

70
tests/unit/responses/test_streaming_iterator_error_events.py
View file

@ -959,3 +959,73 @@ async def test_responses_websocket_turn_failure_reports_the_deployment_it_was_se
assert failure is not None
_assert_names_the_azure_deployment(failure, reported_api_base)
_BEDROCK_SERVICE_UNAVAILABLE_CHUNK: Final = {
"error": {
"message": "The server had an error while processing your request. Sorry about that!",
"type": "server_error",
"code": "service_unavailable",
}
}
def _real_config_iterator(provider: str, model: str, events: tuple[Mapping[str, object], ...]):
payload: Final = b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events)
async def aiter_bytes():
yield payload
response: Final = Mock()
response.headers = {}
response.aiter_bytes = aiter_bytes
logging_obj: Final = Mock(spec=LiteLLMLoggingObj)
logging_obj.model_call_details = {"litellm_params": {}}
logging_obj.completion_start_time = None
config: Final = ProviderConfigManager.get_provider_responses_api_config(
provider=litellm.LlmProviders(provider), model=model
)
assert config is not None
return ResponsesAPIStreamingIterator(
response=response,
model=model,
responses_api_provider_config=config,
logging_obj=logging_obj,
custom_llm_provider=provider,
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("provider", "model"),
[
("bedrock", "us.openai.gpt-5.6-sol"),
("openai", "gpt-5.5"),
("volcengine", "doubao-seed-1-6"),
("bigmodel", "glm-5"),
],
)
async def test_untyped_mid_stream_error_chunk_raises_the_upstream_error(provider: str, model: str):
iterator: Final = _real_config_iterator(
provider,
model,
(
{
"type": "response.output_text.delta",
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"delta": "partial ",
"sequence_number": 1,
},
_BEDROCK_SERVICE_UNAVAILABLE_CHUNK,
),
)
stream: Final = aiter(iterator)
assert (await anext(stream)).type == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA
with pytest.raises(MidStreamFallbackError) as exc_info:
await anext(stream)
assert "The server had an error while processing your request" in str(exc_info.value.original_exception)
assert exc_info.value.status_code >= 500
assert exc_info.value.generated_content == "partial "

13
tests/unit/rust_bridge/host/test_model_capabilities.py
View file

@ -73,3 +73,16 @@ def test_capability_source_observes_runtime_registration_changes(monkeypatch: py
assert before["supports_output_config"] is True
assert after["supports_reasoning"] is False
assert after["supports_output_config"] is False
@pytest.mark.parametrize(
"model, expected",
[
("claude-sonnet-5", True),
("claude-haiku-4-5", False),
("claude-sonnet-5-5", True),
],
)
def test_default_thinking_capability_is_projected(model: str, expected: bool) -> None:
capabilities: Final = anthropic_model_capabilities(model, "anthropic")
assert capabilities["thinking_on_by_default"] is expected

4
tests/unit/rust_bridge/test_detectors.py
View file

@ -51,8 +51,10 @@ def call(**kwargs: object) -> NativeCall:
("openai/gpt-luna-latest", None, True),
("gpt-luna-latest", "openai", True),
("gemini/gemini-pro-latest", None, False),
("gpt-luna-latest", None, False),
("gpt-luna-latest", None, True),
("anthropic/claude-opus-latest", None, True),
("moonshot/kimi-latest", None, True),
("kimi-latest", None, False),
("claude-opus-latest", "anthropic", True),
("", None, False),
),

413
tests/unit/test_lazy_imports.py
View file

@ -1,5 +1,4 @@
"""Simple tests for lazy import functionality."""
import importlib
import os
import subprocess
import sys
@ -7,38 +6,9 @@ from typing import Final
import pytest
import litellm
from litellm._lazy_imports import (
COST_CALCULATOR_NAMES,
LITELLM_LOGGING_NAMES,
UTILS_NAMES,
TOKEN_COUNTER_NAMES,
CACHING_NAMES,
BEDROCK_TYPES_NAMES,
TYPES_UTILS_NAMES,
LLM_CLIENT_CACHE_NAMES,
HTTP_HANDLER_NAMES,
_lazy_import_cost_calculator,
_lazy_import_litellm_logging,
_lazy_import_utils,
_lazy_import_token_counter,
_lazy_import_bedrock_types,
_lazy_import_types_utils,
_lazy_import_caching,
_lazy_import_llm_client_cache,
_lazy_import_http_handlers,
DOTPROMPT_NAMES,
_lazy_import_dotprompt,
LLM_CONFIG_NAMES,
_lazy_import_llm_configs,
TYPES_NAMES,
_lazy_import_types,
LLM_PROVIDER_LOGIC_NAMES,
_lazy_import_llm_provider_logic,
UTILS_MODULE_NAMES,
_lazy_import_utils_module,
)
import litellm.utils
from litellm._lazy_imports_registry import LITELLM_ATTRIBUTES, UTILS_ATTRIBUTES
from tests.test_litellm_rust.support.child_interpreter import run_child_interpreter
@ -58,314 +28,75 @@ def test_import_litellm_does_not_load_fastapi_or_bpe_table():
assert result.stdout.strip() == ""
def _clear_names_from_globals(names: tuple):
"""Clear all names from litellm globals."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in names:
if name in litellm_globals:
del litellm_globals[name]
def _clear_names_from_utils_globals(names: tuple):
"""Clear all names from litellm.utils globals."""
# Get the actual globals dict, not a copy
utils_globals = sys.modules["litellm.utils"].__dict__
for name in names:
if name in utils_globals:
del utils_globals[name]
def _verify_only_requested_name_imported(name: str, all_names: tuple):
"""Verify that only the requested name is in globals, not the others."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for other_name in all_names:
if other_name != name:
assert (
other_name not in litellm_globals
), f"{other_name} should not be imported when importing {name}"
def _verify_only_requested_name_imported_in_utils(name: str, all_names: tuple):
"""Verify that only the requested name is in utils globals, not the others."""
# Get the actual globals dict, not a copy
utils_globals = sys.modules["litellm.utils"].__dict__
for other_name in all_names:
if other_name != name:
assert (
other_name not in utils_globals
), f"{other_name} should not be imported when importing {name}"
def test_cost_calculator_lazy_imports():
"""Test that all cost calculator functions can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
# Test each name individually - only that name should be imported
for name in COST_CALCULATOR_NAMES:
# Clear all names before importing just one
_clear_names_from_globals(COST_CALCULATOR_NAMES)
func = _lazy_import_cost_calculator(name)
assert func is not None
assert callable(func)
assert name in litellm_globals
# Verify only the requested name is in globals, not the others
_verify_only_requested_name_imported(name, COST_CALCULATOR_NAMES)
def test_litellm_logging_lazy_imports():
"""Test that all litellm_logging items can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
# Test each name individually - only that name should be imported
for name in LITELLM_LOGGING_NAMES:
# Clear all names before importing just one
_clear_names_from_globals(LITELLM_LOGGING_NAMES)
item = _lazy_import_litellm_logging(name)
assert item is not None
assert name in litellm_globals
# Verify only the requested name is in globals, not the others
_verify_only_requested_name_imported(name, LITELLM_LOGGING_NAMES)
def test_utils_lazy_imports():
"""Test that all utils functions can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
# Test each name individually - only that name should be imported
for name in UTILS_NAMES:
# Clear all names before importing just one
_clear_names_from_globals(UTILS_NAMES)
attr = _lazy_import_utils(name)
assert attr is not None
assert name in litellm_globals
# Verify only the requested name is in globals, not the others
_verify_only_requested_name_imported(name, UTILS_NAMES)
def test_caching_lazy_imports():
"""Test that all caching classes can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
# Test each name individually - only that name should be imported
for name in CACHING_NAMES:
# Clear all names before importing just one
_clear_names_from_globals(CACHING_NAMES)
cls = _lazy_import_caching(name)
assert cls is not None
assert name in litellm_globals
# Verify only the requested name is in globals, not the others
_verify_only_requested_name_imported(name, CACHING_NAMES)
def test_token_counter_lazy_imports():
"""Test that token counter utilities can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in TOKEN_COUNTER_NAMES:
_clear_names_from_globals(TOKEN_COUNTER_NAMES)
func = _lazy_import_token_counter(name)
assert func is not None
assert name in litellm_globals
_verify_only_requested_name_imported(name, TOKEN_COUNTER_NAMES)
def test_bedrock_types_lazy_imports():
"""Test that Bedrock type aliases can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in BEDROCK_TYPES_NAMES:
_clear_names_from_globals(BEDROCK_TYPES_NAMES)
alias = _lazy_import_bedrock_types(name)
assert alias is not None
assert name in litellm_globals
_verify_only_requested_name_imported(name, BEDROCK_TYPES_NAMES)
def test_types_utils_lazy_imports():
"""Test that common types.utils symbols can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in TYPES_UTILS_NAMES:
_clear_names_from_globals(TYPES_UTILS_NAMES)
obj = _lazy_import_types_utils(name)
assert obj is not None
assert name in litellm_globals
_verify_only_requested_name_imported(name, TYPES_UTILS_NAMES)
def test_llm_client_cache_lazy_imports():
"""Test that LLM client cache class and singleton can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in LLM_CLIENT_CACHE_NAMES:
_clear_names_from_globals(LLM_CLIENT_CACHE_NAMES)
obj = _lazy_import_llm_client_cache(name)
assert obj is not None
assert name in litellm_globals
_verify_only_requested_name_imported(name, LLM_CLIENT_CACHE_NAMES)
def test_http_handler_lazy_imports():
"""Test that HTTP handler singletons can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in HTTP_HANDLER_NAMES:
_clear_names_from_globals(HTTP_HANDLER_NAMES)
handler = _lazy_import_http_handlers(name)
assert handler is not None
assert name in litellm_globals
_verify_only_requested_name_imported(name, HTTP_HANDLER_NAMES)
def test_dotprompt_lazy_imports():
"""Test that dotprompt globals can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in DOTPROMPT_NAMES:
_clear_names_from_globals(DOTPROMPT_NAMES)
obj = _lazy_import_dotprompt(name)
assert name in litellm_globals
# Only the setter must be callable; others may be None by default
if name == "set_global_prompt_directory":
assert callable(obj), f"{name} should be callable"
_verify_only_requested_name_imported(name, DOTPROMPT_NAMES)
def test_unknown_attribute_raises_error():
"""Test that unknown attributes raise AttributeError."""
with pytest.raises(AttributeError):
_lazy_import_cost_calculator("unknown")
with pytest.raises(AttributeError):
_lazy_import_litellm_logging("unknown")
with pytest.raises(AttributeError):
_lazy_import_utils("unknown")
with pytest.raises(AttributeError):
_lazy_import_caching("unknown")
with pytest.raises(AttributeError):
_lazy_import_token_counter("unknown")
with pytest.raises(AttributeError):
_lazy_import_llm_client_cache("unknown")
with pytest.raises(AttributeError):
_lazy_import_bedrock_types("unknown")
with pytest.raises(AttributeError):
_lazy_import_types_utils("unknown")
with pytest.raises(AttributeError):
_lazy_import_llm_configs("unknown")
with pytest.raises(AttributeError):
_lazy_import_types("unknown")
with pytest.raises(AttributeError):
_lazy_import_llm_provider_logic("unknown")
with pytest.raises(AttributeError):
_lazy_import_utils_module("unknown")
def test_llm_config_lazy_imports():
"""Test that LLM config classes can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in LLM_CONFIG_NAMES:
_clear_names_from_globals(LLM_CONFIG_NAMES)
obj = _lazy_import_llm_configs(name)
assert obj is not None
assert name in litellm_globals
# Config classes should be classes/types
assert isinstance(obj, type), f"{name} should be a class"
_verify_only_requested_name_imported(name, LLM_CONFIG_NAMES)
def test_types_lazy_imports():
"""Test that type classes can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in TYPES_NAMES:
_clear_names_from_globals(TYPES_NAMES)
obj = _lazy_import_types(name)
assert obj is not None
assert name in litellm_globals
# Type classes should be classes/types
assert isinstance(obj, type), f"{name} should be a class"
_verify_only_requested_name_imported(name, TYPES_NAMES)
def test_llm_provider_logic_lazy_imports():
"""Test that LLM provider logic functions can be lazy imported."""
# Get the actual globals dict, not a copy
litellm_globals = sys.modules["litellm"].__dict__
for name in LLM_PROVIDER_LOGIC_NAMES:
_clear_names_from_globals(LLM_PROVIDER_LOGIC_NAMES)
func = _lazy_import_llm_provider_logic(name)
assert func is not None
assert callable(func)
assert name in litellm_globals
_verify_only_requested_name_imported(name, LLM_PROVIDER_LOGIC_NAMES)
def test_utils_module_lazy_imports():
"""Test that utils module attributes can be lazy imported."""
# Get the actual globals dict, not a copy
utils_globals = sys.modules["litellm.utils"].__dict__
for name in UTILS_MODULE_NAMES:
_clear_names_from_utils_globals(UTILS_MODULE_NAMES)
obj = _lazy_import_utils_module(name)
assert obj is not None
assert name in utils_globals
_verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES)
def _declared(location: tuple[str, str]) -> object:
module_path, attr_name = location
return getattr(importlib.import_module(module_path, package="litellm"), attr_name)
@pytest.mark.parametrize("name", sorted(LITELLM_ATTRIBUTES))
def test_litellm_attribute_is_the_declared_object(name: str):
assert getattr(litellm, name) is _declared(LITELLM_ATTRIBUTES[name])
@pytest.mark.parametrize("name", sorted(UTILS_ATTRIBUTES))
def test_utils_attribute_is_the_same_object_from_litellm_and_litellm_utils(name: str):
expected: Final = _declared(UTILS_ATTRIBUTES[name])
assert getattr(litellm.utils, name) is expected
assert getattr(litellm, name) is expected
@pytest.mark.parametrize(
("instance_name", "class_name"),
[
("in_memory_llm_clients_cache", "LLMClientCache"),
("openaiOSeriesConfig", "OpenAIOSeriesConfig"),
("openAIGPTConfig", "OpenAIGPTConfig"),
("openAIGPTAudioConfig", "OpenAIGPTAudioConfig"),
("openAIGPT5Config", "OpenAIGPT5Config"),
("nvidiaNimConfig", "NvidiaNimConfig"),
("nvidiaNimEmbeddingConfig", "NvidiaNimEmbeddingConfig"),
("priority_reservation_settings", "PriorityReservationSettings"),
("logging_callback_manager", "LoggingCallbackManager"),
],
)
def test_lazy_instance_is_a_single_instance_of_its_class(instance_name: str, class_name: str):
instance: Final = getattr(litellm, instance_name)
assert type(instance) is getattr(litellm, class_name)
assert getattr(litellm, instance_name) is instance
def test_module_level_http_clients_use_request_timeout(monkeypatch: pytest.MonkeyPatch):
client_names: Final = ("module_level_client", "module_level_aclient")
monkeypatch.setattr(litellm, "request_timeout", 7)
for client_name in client_names:
monkeypatch.delitem(vars(litellm), client_name, raising=False)
try:
assert litellm.module_level_client.timeout == 7
assert litellm.module_level_aclient.timeout == 7
assert litellm.module_level_client is litellm.module_level_client
finally:
for client_name in client_names:
vars(litellm).pop(client_name, None)
def test_provider_list_lists_every_provider():
assert litellm.provider_list == list(litellm.LlmProviders)
@pytest.mark.parametrize("name", ["proxy", "guardrails", "harness", "_service_logger"])
def test_submodule_attribute_is_the_submodule(name: str):
assert getattr(litellm, name) is importlib.import_module(f"litellm.{name}")
@pytest.mark.parametrize("module", [litellm, litellm.utils])
def test_unknown_attribute_raises_attribute_error(module: object):
with pytest.raises(AttributeError, match="not_a_litellm_attribute"):
getattr(module, "not_a_litellm_attribute")
assert not hasattr(module, "not_a_litellm_attribute")
def test_litellm_instances_are_not_attributes_of_litellm_utils():
assert not hasattr(litellm.utils, "module_level_client")
@pytest.mark.parametrize(

15
tests/unit/test_main.py
View file

@ -8,7 +8,7 @@ import logging
import os
import threading
import urllib.parse
from collections.abc import Callable, Mapping
from collections.abc import Awaitable, Callable, Mapping
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from datetime import datetime
@ -9395,6 +9395,19 @@ async def test_responses_with_retries(sync_mode):
assert mock_responses.call_args.kwargs["max_retries"] == 0
@pytest.mark.parametrize("retry_function", [acompletion_with_retries, aresponses_with_retries])
@pytest.mark.asyncio
async def test_async_retry_helpers_stop_when_the_call_is_cancelled(
retry_function: Callable[..., Awaitable[object]],
) -> None:
upstream: Final = AsyncMock(side_effect=[asyncio.CancelledError(), MagicMock()])
with pytest.raises(asyncio.CancelledError):
await retry_function(model="gpt-4o", num_retries=3, original_function=upstream)
assert upstream.await_count == 1
def test_azure_embedding_exceptions():
with pytest.raises(Exception, match="Mock error") as exc_info:
litellm.embedding(

1
tests/unit/test_private_usage_aliases.py
View file

@ -1185,7 +1185,6 @@ ALIAS_CASES: Final = (
("litellm", "", "_turn_on_debug", "turn_on_debug", False),
("litellm", "", "_turn_on_json", "turn_on_json", False),
("litellm._lazy_imports", "", "_get_default_encoding", "get_default_encoding", False),
("litellm._lazy_imports", "", "_get_lazy_import_registry", "get_lazy_import_registry", False),
("litellm._lazy_imports", "", "_get_messages_reach_token_count", "get_messages_reach_token_count", False),
("litellm._lazy_imports", "", "_get_token_counter_new", "get_token_counter_new", False),
("litellm._logging", "", "_is_debugging_on", "is_debugging_on", False),

121
tests/unit/test_utils.py
View file

@ -18,6 +18,7 @@ from concurrent.futures import Future, ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from functools import partial
from pathlib import Path, PurePath
from types import MappingProxyType
from typing import Final, SupportsIndex, cast
from unittest import mock
from unittest.mock import AsyncMock, MagicMock, Mock, patch
@ -1028,6 +1029,8 @@ _COST_MAP_ROW_SCHEMA: Final = {
"audio_transcription_config": {"type": "string"},
"deprecation_date": {"type": "string"},
"successor_model": {"type": "string"},
"aliases": {"type": "array", "items": {"type": "string"}},
"alias_of": {"type": "string"},
"deprecation_source": {"type": "string"},
"input_cost_per_audio_per_second": {"type": "number"},
"input_cost_per_audio_per_second_above_128k_tokens": {"type": "number"},
@ -3871,6 +3874,124 @@ class TestGetOptionalParamsSpeedGate:
assert result.get("speed") == "fast"
class TestGetOptionalParamsOutputConfigGate:
"""Regression for the issue 24549 follow-up (SLP-2948): Anthropic's ``output_config`` is not a named
OpenAI param, so it rode the provider-specific pass-through into ``extra_body`` and ``drop_params`` never
saw it."""
UNREAD_LEGS: Final = (
("openai", "gpt-5.6"),
("azure", "gpt-5.6"),
("xiaomi_mimo", "mimo-v2.6-pro"),
("hosted_vllm", "my-model"),
("gemini", "gemini-3-pro-preview"),
("bedrock", "us.amazon.nova-pro-v1:0"),
)
CLAUDE_LEGS: Final = (
("anthropic", "claude-opus-4-8"),
("vertex_ai", "claude-opus-4-8"),
("azure_ai", "claude-opus-4-8"),
("bedrock", "converse/anthropic.claude-opus-4-8-v1:0"),
("bedrock", "invoke/anthropic.claude-opus-4-8-v1:0"),
)
OUTPUT_CONFIG: Final = MappingProxyType({"effort": "medium"})
@staticmethod
def _output_config_anywhere(optional_params: Mapping[str, object]) -> bool:
extra_body: Final = optional_params.get("extra_body")
return "output_config" in optional_params or (isinstance(extra_body, dict) and "output_config" in extra_body)
@pytest.mark.parametrize("global_drop_params", [True, False])
@pytest.mark.parametrize(("custom_llm_provider", "model"), UNREAD_LEGS)
def test_drop_params_strips_output_config_on_providers_that_do_not_read_it(
self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str, global_drop_params: bool
) -> None:
monkeypatch.setattr(litellm, "drop_params", global_drop_params)
result: Final = litellm.get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
output_config=dict(self.OUTPUT_CONFIG),
drop_params=not global_drop_params,
)
assert not self._output_config_anywhere(result), result
def test_drop_params_keeps_the_mapped_reasoning_effort_beside_the_dropped_output_config(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model="gpt-5.6",
custom_llm_provider="openai",
output_config=dict(self.OUTPUT_CONFIG),
reasoning_effort="low",
drop_params=True,
)
assert result.get("reasoning_effort") == "low"
assert not self._output_config_anywhere(result), result
@pytest.mark.parametrize(("custom_llm_provider", "model"), [("openai", "gpt-5.6"), ("hosted_vllm", "my-model")])
def test_output_config_is_forwarded_untouched_without_drop_params(
self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
output_config=dict(self.OUTPUT_CONFIG),
drop_params=False,
)
assert self._output_config_anywhere(result), result
@pytest.mark.parametrize(("custom_llm_provider", "model"), CLAUDE_LEGS)
def test_drop_params_keeps_output_config_for_claude_transformations(
self, monkeypatch: pytest.MonkeyPatch, custom_llm_provider: str, model: str
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model=model,
custom_llm_provider=custom_llm_provider,
output_config=dict(self.OUTPUT_CONFIG),
drop_params=True,
)
assert self._output_config_anywhere(result), result
def test_drop_params_keeps_output_config_for_a_downstream_litellm_proxy(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model="claude-opus-4-8",
custom_llm_provider="litellm_proxy",
output_config=dict(self.OUTPUT_CONFIG),
drop_params=True,
)
assert self._output_config_anywhere(result), result
def test_drop_params_keeps_output_config_for_a_deployment_that_declares_reasoning_effort_levels(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model="my-kimi",
custom_llm_provider="hosted_vllm",
output_config=dict(self.OUTPUT_CONFIG),
model_info={"id": "kimi-vllm-1", "reasoning_effort_levels": ["low", "medium", "high"]},
drop_params=True,
)
assert self._output_config_anywhere(result), result
def test_allowed_openai_params_overrides_the_drop(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "drop_params", False)
result: Final = litellm.get_optional_params(
model="gpt-5.6",
custom_llm_provider="openai",
output_config=dict(self.OUTPUT_CONFIG),
allowed_openai_params=["output_config"],
drop_params=True,
)
assert self._output_config_anywhere(result), result
class TestGetOptionalParamsDeepSeek:
"""Tests that deepseek provider uses DeepSeekChatConfig for parameter mapping."""

44
ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.integration.test.tsx Normal file
View file

@ -0,0 +1,44 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { act, render, screen } from "@testing-library/react";
import { useState } from "react";
import { describe, expect, it, vi } from "vitest";
import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
vi.mock("../networking", () => ({
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
modelInfoCall: vi.fn().mockResolvedValue({ data: [] }),
}));
vi.mock("@/components/llm_calls/fetch_models", () => ({
fetchAvailableModels: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]),
fetchAvailableModelsForTeam: vi.fn().mockResolvedValue([{ model_group: "gpt-5" }]),
}));
const EchoingCreateForm = () => {
const [value, setValue] = useState<RouterSettingsAccordionValue | undefined>(undefined);
return <RouterSettingsAccordion accessToken="test-token" value={value} onChange={setValue} />;
};
describe("RouterSettingsAccordion on a create form", () => {
it("should render every reliability input after the parent echoes the first value back", async () => {
render(
<QueryClientProvider client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}>
<EchoingCreateForm />
</QueryClientProvider>,
);
await act(async () => {
await new Promise((resolve) => setTimeout(resolve, 250));
});
for (const field of [
"allowed_fails",
"cooldown_time",
"num_retries",
"timeout",
"retry_after",
"model_group_alias",
]) {
expect(screen.getByRole("textbox", { name: new RegExp(field) })).toBeEnabled();
}
});
});

189
ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.test.tsx
View file

@ -1,13 +1,18 @@
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
import type { ReactElement, ReactNode } from "react";
import { createRef, useState, type ReactElement, type ReactNode } from "react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { RouterSettingsFormValue } from "../router_settings/RouterSettingsForm";
import { fetchAvailableModels, fetchAvailableModelsForTeam } from "@/components/llm_calls/fetch_models";
import RouterSettingsAccordion, { RouterSettingsAccordionValue } from "./RouterSettingsAccordion";
import { getRouterSettingsCall, modelInfoCall } from "../networking";
import RouterSettingsAccordion, {
RouterSettingsAccordionRef,
RouterSettingsAccordionValue,
} from "./RouterSettingsAccordion";
vi.mock("../networking", () => ({
getRouterSettingsCall: vi.fn().mockResolvedValue({}),
modelInfoCall: vi.fn(),
}));
vi.mock("@/components/llm_calls/fetch_models", () => ({
@ -130,6 +135,186 @@ describe("RouterSettingsAccordion", () => {
expect(screen.getByTestId("available-models")).not.toHaveTextContent("slow-team-model");
});
describe("deployment weights", () => {
const gpt5Deployments = {
data: [
{ model_name: "gpt-5", litellm_params: { model: "openai/gpt-5" }, model_info: { id: "dep-openai" } },
{ model_name: "gpt-5", litellm_params: { model: "azure/gpt-5" }, model_info: { id: "dep-azure" } },
],
};
beforeEach(() => {
vi.mocked(modelInfoCall).mockResolvedValue(gpt5Deployments);
});
it("should save stored weights together with a newly entered one", async () => {
const ref = createRef<RouterSettingsAccordionRef>();
renderWithQueryClient(
<RouterSettingsAccordion
ref={ref}
accessToken="test-token"
teamId="team-123"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
fireEvent.change(await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)"), { target: { value: "1" } });
expect(await screen.findByText("90% of traffic")).toBeInTheDocument();
expect(screen.getByText("10% of traffic")).toBeInTheDocument();
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({
"gpt-5": { "dep-openai": 9, "dep-azure": 1 },
});
expect(vi.mocked(modelInfoCall).mock.calls[0][7]).toBe("team-123");
expect(vi.mocked(modelInfoCall).mock.calls[0][11]).toBe("gpt-5");
});
it("should save null once every weight is cleared, so the stored weights are removed", async () => {
const ref = createRef<RouterSettingsAccordionRef>();
renderWithQueryClient(
<RouterSettingsAccordion
ref={ref}
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } });
await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull());
});
it("should keep a model group on screen when the parent echoes back the value it was sent", async () => {
const EchoingParent = () => {
const [value, setValue] = useState<RouterSettingsAccordionValue>({
router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } },
});
return <RouterSettingsAccordion accessToken="test-token" value={value} onChange={setValue} />;
};
renderWithQueryClient(<EchoingParent />);
fireEvent.change(await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)"), { target: { value: "" } });
await act(async () => {
vi.advanceTimersByTime(200);
});
expect(screen.getByLabelText("Weight for openai/gpt-5 (dep-openai)")).toHaveValue("");
expect(screen.getByRole("button", { name: "Remove weights for gpt-5" })).toBeInTheDocument();
});
it("should say the deployments failed to load instead of calling saved weights stale", async () => {
vi.mocked(modelInfoCall).mockRejectedValue(new Error("403"));
const ref = createRef<RouterSettingsAccordionRef>();
renderWithQueryClient(
<RouterSettingsAccordion
ref={ref}
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
expect(
await screen.findByText("Could not load this model group's deployments. Saved weights are kept."),
).toBeInTheDocument();
expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument();
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } });
});
it("should warn that weights are ignored when the inherited global strategy is not simple-shuffle", async () => {
vi.mocked(getRouterSettingsCall).mockResolvedValueOnce({
fields: [{ field_name: "routing_strategy", options: ["simple-shuffle"], field_value: "latency-based-routing" }],
});
renderWithQueryClient(
<RouterSettingsAccordion
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
expect(
await screen.findByText("Weights are ignored with latency-based-routing. Select simple-shuffle to use them."),
).toBeInTheDocument();
});
it("should accept a decimal weight typed one character at a time", async () => {
const ref = createRef<RouterSettingsAccordionRef>();
renderWithQueryClient(
<RouterSettingsAccordion
ref={ref}
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
const input = await screen.findByLabelText("Weight for azure/gpt-5 (dep-azure)");
fireEvent.change(input, { target: { value: "1." } });
expect(input).toHaveValue("1.");
fireEvent.change(input, { target: { value: "1.5" } });
expect(input).toHaveValue("1.5");
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({
"gpt-5": { "dep-openai": 9, "dep-azure": 1.5 },
});
});
it("should ignore keystrokes that would not save as a weight, so the field always shows the saved value", async () => {
const ref = createRef<RouterSettingsAccordionRef>();
renderWithQueryClient(
<RouterSettingsAccordion
ref={ref}
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
const input = await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)");
fireEvent.change(input, { target: { value: "9x" } });
fireEvent.change(input, { target: { value: "-1" } });
expect(input).toHaveValue("9");
expect(ref.current?.getValue().router_settings.weights).toStrictEqual({ "gpt-5": { "dep-openai": 9 } });
});
it("should not call saved weights stale while the deployments are still loading", async () => {
vi.mocked(modelInfoCall).mockReturnValue(new Promise(() => {}));
renderWithQueryClient(
<RouterSettingsAccordion
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } }}
/>,
);
expect(await screen.findByText("dep-openai")).toBeInTheDocument();
expect(screen.queryByText("Deployment no longer exists")).not.toBeInTheDocument();
});
it("should drop weights picked for one team when the form switches to another team", async () => {
const ref = createRef<RouterSettingsAccordionRef>();
const value = { router_settings: { weights: { "gpt-5": { "dep-openai": 9 } } } };
const { rerender } = renderWithQueryClient(
<RouterSettingsAccordion ref={ref} accessToken="test-token" teamId="team-a" value={value} />,
);
await screen.findByLabelText("Weight for openai/gpt-5 (dep-openai)");
rerender(<RouterSettingsAccordion ref={ref} accessToken="test-token" teamId="team-b" value={value} />);
await waitFor(() => expect(ref.current?.getValue().router_settings.weights).toBeNull());
});
it("should flag a group whose weights are all zero", async () => {
renderWithQueryClient(
<RouterSettingsAccordion
accessToken="test-token"
value={{ router_settings: { weights: { "gpt-5": { "dep-openai": 0 } } } }}
/>,
);
expect(
await screen.findByText("Set at least one weight above 0, or remove this model group."),
).toBeInTheDocument();
});
});
it("does not call onChange when unmounted mid-wait", async () => {
const onChange = vi.fn<(value: RouterSettingsAccordionValue) => void>();
const { unmount } = renderWithQueryClient(<RouterSettingsAccordion accessToken="test-token" onChange={onChange} />);

58
ui/litellm-dashboard/src/components/common_components/RouterSettingsAccordion.tsx
View file

@ -9,6 +9,8 @@ import { Fallbacks } from "../Settings/RouterSettings/Fallbacks/AddFallbacks";
import { FallbackSelectionForm } from "../Settings/RouterSettings/Fallbacks/FallbackSelectionForm";
import { FallbackGroup } from "../Settings/RouterSettings/Fallbacks/FallbackGroupConfig";
import { fetchAvailableModels, fetchAvailableModelsForTeam, ModelGroup } from "@/components/llm_calls/fetch_models";
import RouterWeightsSection from "./RouterWeightsSection";
import { RouterWeights, routerWeightsFrom, savedRouterWeights } from "./routerWeights";
export interface RouterSettingsAccordionValue {
router_settings: {
@ -24,6 +26,7 @@ export interface RouterSettingsAccordionValue {
model_group_alias?: { [key: string]: any } | null;
enable_tag_filtering?: boolean;
routing_strategy_args?: { [key: string]: any } | null;
weights?: RouterWeights | null;
};
}
@ -41,20 +44,38 @@ export interface RouterSettingsAccordionRef {
const PROPAGATE_WAIT_MS = 100;
const EMPTY_RELIABILITY_SETTINGS = {
allowed_fails: null,
cooldown_time: null,
num_retries: null,
timeout: null,
retry_after: null,
model_group_alias: null,
};
const valueKeyOf = (settings: RouterSettingsAccordionValue["router_settings"]): string => {
const { routing_strategy, fallbacks, enable_tag_filtering, weights } = settings;
const identity = { routing_strategy, fallbacks, enable_tag_filtering, weights };
return JSON.stringify(identity);
};
const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSettingsAccordionProps>(
({ accessToken, value, onChange, modelData, teamId }, ref) => {
const { t } = useTranslation();
const [formValue, setFormValue] = useState<RouterSettingsFormValue>({
routerSettings: {},
routerSettings: EMPTY_RELIABILITY_SETTINGS,
selectedStrategy: null,
enableTagFiltering: false,
});
const [fallbacks, setFallbacks] = useState<Fallbacks>([]);
const [fallbackGroups, setFallbackGroups] = useState<FallbackGroup[]>([]);
const [weights, setWeights] = useState<RouterWeights>({});
const [availableRoutingStrategies, setAvailableRoutingStrategies] = useState<string[]>([]);
const [globalRoutingStrategy, setGlobalRoutingStrategy] = useState<string | null>(null);
const [routerFieldsMetadata, setRouterFieldsMetadata] = useState<{ [key: string]: any }>({});
const [routingStrategyDescriptions, setRoutingStrategyDescriptions] = useState<{ [key: string]: string }>({});
const isInternalUpdateRef = useRef(false);
const weightsTeamIdRef = useRef(teamId);
const lastInitializedValueRef = useRef<string | null>(null);
// Convert fallbacks format to groups format
@ -90,13 +111,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
// Initialize from value prop if provided (only when value actually changes externally)
useEffect(() => {
// Create a stable key from the value to detect actual external changes
const valueKey = value?.router_settings
? JSON.stringify({
routing_strategy: value.router_settings.routing_strategy,
fallbacks: value.router_settings.fallbacks,
enable_tag_filtering: value.router_settings.enable_tag_filtering,
})
: null;
const valueKey = value?.router_settings ? valueKeyOf(value.router_settings) : null;
// Skip if this is an internal update (from our own onChange) and the value hasn't actually changed
if (isInternalUpdateRef.current && valueKey === lastInitializedValueRef.current) {
@ -118,7 +133,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
if (value?.router_settings) {
const rs = value.router_settings;
const { fallbacks: _, ...routerSettingsWithoutFallbacks } = rs;
const { fallbacks: _, weights: storedWeights, ...routerSettingsWithoutFallbacks } = rs;
setFormValue({
routerSettings: routerSettingsWithoutFallbacks as { [key: string]: any },
selectedStrategy: rs.routing_strategy || null,
@ -127,10 +142,11 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
const initialFallbacks = rs.fallbacks || [];
setFallbacks(initialFallbacks);
setFallbackGroups(fallbacksToGroups(initialFallbacks));
setWeights(routerWeightsFrom(storedWeights));
} else {
// Initialize with empty defaults if no value provided
setFormValue({
routerSettings: {},
routerSettings: EMPTY_RELIABILITY_SETTINGS,
selectedStrategy: null,
enableTagFiltering: false,
});
@ -142,9 +158,16 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
fallbackModels: [],
},
]);
setWeights({});
}
}, [value]);
useEffect(() => {
if (weightsTeamIdRef.current === teamId) return;
weightsTeamIdRef.current = teamId;
setWeights({});
}, [teamId]);
// Fetch router settings metadata
useEffect(() => {
if (!accessToken) {
@ -169,6 +192,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
if (routingStrategyField?.options) {
setAvailableRoutingStrategies(routingStrategyField.options);
}
setGlobalRoutingStrategy(routingStrategyField?.field_value ?? null);
// Store routing strategy descriptions
if (data.routing_strategy_descriptions) {
@ -297,6 +321,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
model_group_alias: normalizeValue(updatedVariables.model_group_alias),
enable_tag_filtering: formValue.enableTagFiltering,
routing_strategy_args: normalizeValue(updatedVariables.routing_strategy_args),
weights: savedRouterWeights(weights),
};
};
@ -307,6 +332,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
}
isInternalUpdateRef.current = true;
const finalRouterSettings = buildRouterSettings();
lastInitializedValueRef.current = valueKeyOf(finalRouterSettings);
onChange({
router_settings: finalRouterSettings,
});
@ -321,7 +347,7 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
}
debouncedPropagate();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [formValue, fallbacks]);
}, [formValue, fallbacks, weights]);
const handleFallbackGroupsChange = (newGroups: FallbackGroup[]): void => {
setFallbackGroups(newGroups);
@ -360,6 +386,16 @@ const RouterSettingsAccordion = forwardRef<RouterSettingsAccordionRef, RouterSet
availableRoutingStrategies={availableRoutingStrategies}
routingStrategyDescriptions={routingStrategyDescriptions}
/>
<div className="mt-8 border-t border-border pt-8">
<RouterWeightsSection
accessToken={accessToken}
teamId={teamId}
availableModels={availableModels}
selectedStrategy={formValue.selectedStrategy ?? globalRoutingStrategy}
weights={weights}
onChange={setWeights}
/>
</div>
</TabsContent>
<TabsContent value="2" keepMounted>
<FallbackSelectionForm

9
ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.test.tsx
View file

@ -20,6 +20,15 @@ describe("RouterSettingsSummary", () => {
expect(screen.getByText("Number of Retries: 3")).toBeInTheDocument();
});
it("should list each weighted model group with its deployment weights", () => {
render(<RouterSettingsSummary routerSettings={{ weights: { "gpt-5": { "dep-a": 9, "dep-b": 1 }, empty: {} } }} />);
expect(screen.getByText("Deployment Weights:")).toBeInTheDocument();
expect(screen.getByText("gpt-5")).toBeInTheDocument();
expect(screen.getByText("dep-a: 9, dep-b: 1")).toBeInTheDocument();
expect(screen.queryByText("empty")).not.toBeInTheDocument();
});
it("should show the empty state when every setting is null", () => {
render(<RouterSettingsSummary routerSettings={{ fallbacks: null, num_retries: null }} />);

24
ui/litellm-dashboard/src/components/common_components/RouterSettingsSummary.tsx
View file

@ -2,6 +2,7 @@ import { useTranslation } from "react-i18next";
import { Badge } from "@/components/ui/badge";
import i18n from "@/i18n/i18n";
import { hasRouterSettings } from "./routerSettingsPayload";
import { routerWeightsFrom, savedRouterWeights } from "./routerWeights";
interface RouterSettingsSummaryProps {
routerSettings: Record<string, unknown> | null | undefined;
@ -15,6 +16,28 @@ const fallbackEntries = (fallbacks: unknown): Array<[string, string[]]> => {
);
};
export function RouterWeightsSummary({ weights }: { weights: unknown }) {
const { t } = useTranslation();
const saved = savedRouterWeights(routerWeightsFrom(weights));
if (!saved) return null;
return (
<div>
<div>{t("routerWeights.summary")}</div>
<div className="mt-1 space-y-1">
{Object.entries(saved).map(([group, groupWeights]) => (
<div key={group} className="text-xs text-muted-foreground">
<span className="font-medium">{group}</span>
<span className="mx-1 text-muted-foreground">-&gt;</span>
{Object.entries(groupWeights)
.map(([deploymentId, weight]) => `${deploymentId}: ${weight}`)
.join(", ")}
</div>
))}
</div>
</div>
);
}
export default function RouterSettingsSummary({
routerSettings,
emptyText = i18n.t("routerSettingsSummary.empty"),
@ -63,6 +86,7 @@ export default function RouterSettingsSummary({
</div>
</div>
)}
<RouterWeightsSummary weights={settings.weights} />
</div>
);
}

206
ui/litellm-dashboard/src/components/common_components/RouterWeightsSection.tsx Normal file
View file

@ -0,0 +1,206 @@
import { useQuery } from "@tanstack/react-query";
import { AlertCircle, X } from "lucide-react";
import { useId, useState } from "react";
import { useTranslation } from "react-i18next";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { SearchSelect } from "@/components/shared/SearchSelect";
import { modelInfoCall } from "../networking";
import {
GroupWeights,
RouterWeights,
WeightableDeployment,
hasNoPositiveWeight,
parseWeightInput,
weightRows,
withDeploymentWeight,
withGroup,
withoutGroup,
} from "./routerWeights";
const WEIGHTED_STRATEGY = "simple-shuffle";
const DEPLOYMENTS_PAGE_SIZE = 1000;
interface RouterWeightsSectionProps {
accessToken: string;
teamId?: string | null;
availableModels: string[];
selectedStrategy: string | null;
weights: RouterWeights;
onChange: (weights: RouterWeights) => void;
}
interface WeightGroupProps {
accessToken: string;
teamId?: string | null;
group: string;
groupWeights: GroupWeights;
onWeightChange: (deploymentId: string, weight: number | null) => void;
onRemove: () => void;
}
interface WeightInputProps {
label: string;
weight: number | null;
onChange: (weight: number | null) => void;
}
const formatWeight = (weight: number | null): string => (weight === null ? "" : String(weight));
function WeightInput({ label, weight, onChange }: WeightInputProps) {
const [draft, setDraft] = useState(formatWeight(weight));
const [shownWeight, setShownWeight] = useState(weight);
if (weight !== shownWeight) {
setShownWeight(weight);
if (parseWeightInput(draft) !== weight) setDraft(formatWeight(weight));
}
const handleChange = (event: React.ChangeEvent<HTMLInputElement>) => {
const parsed = parseWeightInput(event.target.value);
if (parsed === undefined) return;
setDraft(event.target.value);
onChange(parsed);
};
return (
<Input type="text" inputMode="decimal" className="w-28" aria-label={label} value={draft} onChange={handleChange} />
);
}
function WeightGroup({ accessToken, teamId, group, groupWeights, onWeightChange, onRemove }: WeightGroupProps) {
const { t } = useTranslation();
const {
data: deployments = [],
isLoading,
isError,
} = useQuery<WeightableDeployment[]>({
queryKey: ["routerWeightDeployments", accessToken, teamId ?? null, group],
queryFn: async () =>
(
await modelInfoCall(
accessToken,
"",
"",
1,
DEPLOYMENTS_PAGE_SIZE,
undefined,
undefined,
teamId ?? undefined,
undefined,
undefined,
undefined,
group,
)
).data ?? [],
enabled: Boolean(accessToken),
});
const rows = weightRows(group, groupWeights, deployments, teamId ?? null);
const deploymentsKnown = !isLoading && !isError;
const noDeployments = deploymentsKnown && rows.length === 0;
return (
<div className="rounded-md border border-border p-4 space-y-3">
<div className="flex items-center justify-between">
<span className="text-sm font-semibold text-foreground">{group}</span>
<Button
type="button"
variant="ghost"
size="icon-xs"
aria-label={t("routerWeights.removeGroup", { group })}
onClick={onRemove}
>
<X />
</Button>
</div>
{isError && (
<p role="alert" className="text-xs text-destructive">
{t("routerWeights.loadFailed")}
</p>
)}
{noDeployments && <p className="text-xs text-muted-foreground">{t("routerWeights.noDeployments")}</p>}
{rows.map((row) => (
<div key={row.deploymentId} className="flex items-center gap-3">
<div className="min-w-0 flex-1">
<div className="truncate text-sm text-foreground">
{row.label ?? (deploymentsKnown ? t("routerWeights.unknownDeployment") : "")}
</div>
<div className="truncate text-xs text-muted-foreground">{row.deploymentId}</div>
</div>
<WeightInput
label={t("routerWeights.weightFor", {
deployment: row.label ? `${row.label} (${row.deploymentId})` : row.deploymentId,
})}
weight={row.weight}
onChange={(weight) => onWeightChange(row.deploymentId, weight)}
/>
<span className="w-24 text-right text-xs text-muted-foreground">
{row.sharePercent === null ? "" : t("routerWeights.share", { percent: row.sharePercent })}
</span>
</div>
))}
{hasNoPositiveWeight(groupWeights) && (
<p role="alert" className="flex items-center gap-2 text-xs text-destructive">
<AlertCircle className="size-4" />
{t("routerWeights.noPositiveWeight")}
</p>
)}
</div>
);
}
export default function RouterWeightsSection({
accessToken,
teamId,
availableModels,
selectedStrategy,
weights,
onChange,
}: RouterWeightsSectionProps) {
const { t } = useTranslation();
const addGroupInputId = useId();
const unweightedModels = availableModels.filter((model) => !(model in weights));
const ignoredByStrategy = Boolean(selectedStrategy) && selectedStrategy !== WEIGHTED_STRATEGY;
return (
<div className="space-y-4">
<div className="max-w-3xl">
<h3 className="text-sm font-medium text-foreground">{t("routerWeights.title")}</h3>
<p className="mt-1 text-xs text-muted-foreground">{t("routerWeights.description")}</p>
</div>
{ignoredByStrategy && Object.keys(weights).length > 0 && (
<div className="flex items-center gap-2 rounded-sm bg-warning/10 p-2 text-xs text-warning">
<AlertCircle className="size-4" />
<span>{t("routerWeights.ignoredForStrategy", { strategy: selectedStrategy })}</span>
</div>
)}
{Object.entries(weights).map(([group, groupWeights]) => (
<WeightGroup
key={group}
accessToken={accessToken}
teamId={teamId}
group={group}
groupWeights={groupWeights}
onWeightChange={(deploymentId, weight) =>
onChange(withDeploymentWeight(weights, group, deploymentId, weight))
}
onRemove={() => onChange(withoutGroup(weights, group))}
/>
))}
<div className="max-w-md">
<label htmlFor={addGroupInputId} className="mb-2 block text-sm font-medium text-foreground">
{t("routerWeights.addGroup")}
</label>
<SearchSelect
inputId={addGroupInputId}
options={unweightedModels.map((model) => ({ label: model, value: model }))}
value={null}
onValueChange={(group) => {
if (group) onChange(withGroup(weights, group));
}}
placeholder={t("routerWeights.addGroupPlaceholder")}
emptyText={t("routerWeights.noModels")}
/>
</div>
</div>
);
}

9
ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.test.ts
View file

@ -101,9 +101,18 @@ describe("routerSettingsUpdate", () => {
model_group_alias: null,
enable_tag_filtering: null,
routing_strategy_args: null,
weights: null,
});
});
it("should replace stored weights with the edited ones, including clearing them", () => {
const stored = { weights: { "gpt-4": { "dep-a": 9, "dep-b": 1 } }, num_retries: 2 };
expect(routerSettingsUpdate({ num_retries: 2, weights: { "gpt-4": { "dep-a": 1 } } }, stored)).toMatchObject({
weights: { "gpt-4": { "dep-a": 1 } },
});
expect(routerSettingsUpdate({ num_retries: 2, weights: null }, stored)).toMatchObject({ weights: null });
});
it("should send the edited settings when the user configured something", () => {
expect(routerSettingsUpdate({ fallbacks }, null)).toMatchObject({ fallbacks });
});

1
ui/litellm-dashboard/src/components/common_components/routerSettingsPayload.ts
View file

@ -15,6 +15,7 @@ const EDITOR_OWNED_FIELDS: Record<keyof RouterSettings, true> = {
model_group_alias: true,
enable_tag_filtering: true,
routing_strategy_args: true,
weights: true,
};
const EDITOR_OWNED_KEYS = Object.keys(EDITOR_OWNED_FIELDS) as Array<keyof RouterSettings>;

152
ui/litellm-dashboard/src/components/common_components/routerWeights.test.ts Normal file
View file

@ -0,0 +1,152 @@
import { describe, expect, it } from "vitest";
import {
hasNoPositiveWeight,
parseWeightInput,
routerWeightsFrom,
savedRouterWeights,
weightRows,
withDeploymentWeight,
withGroup,
withoutGroup,
} from "./routerWeights";
const deployment = (id: string, modelName: string, model: string, team?: { id: string; publicName: string }) => ({
model_name: modelName,
litellm_params: { model },
model_info: { id, ...(team ? { team_id: team.id, team_public_model_name: team.publicName } : {}) },
});
describe("routerWeightsFrom", () => {
it("should keep well-formed groups and drop malformed weights from the stored blob", () => {
expect(
routerWeightsFrom({
"gpt-5": { a: 9, b: 1, bad: "1", negative: -1, inf: Infinity },
broken: [1, 2],
}),
).toStrictEqual({ "gpt-5": { a: 9, b: 1 } });
});
it("should read missing or non-object weights as none", () => {
expect(routerWeightsFrom(undefined)).toStrictEqual({});
expect(routerWeightsFrom(null)).toStrictEqual({});
expect(routerWeightsFrom("x")).toStrictEqual({});
});
});
describe("weight edits", () => {
it("should set, replace and clear one deployment without touching others", () => {
const start = { "gpt-5": { a: 9 }, other: { c: 2 } };
const added = withDeploymentWeight(start, "gpt-5", "b", 1);
expect(added).toStrictEqual({ "gpt-5": { a: 9, b: 1 }, other: { c: 2 } });
expect(withDeploymentWeight(added, "gpt-5", "a", 3)).toStrictEqual({ "gpt-5": { a: 3, b: 1 }, other: { c: 2 } });
expect(withDeploymentWeight(added, "gpt-5", "a", null)).toStrictEqual({ "gpt-5": { b: 1 }, other: { c: 2 } });
expect(start).toStrictEqual({ "gpt-5": { a: 9 }, other: { c: 2 } });
});
it("should add an empty group once and remove a group entirely", () => {
const withEmpty = withGroup({ "gpt-5": { a: 9 } }, "claude");
expect(withEmpty).toStrictEqual({ "gpt-5": { a: 9 }, claude: {} });
expect(withGroup(withEmpty, "gpt-5")).toBe(withEmpty);
expect(withoutGroup(withEmpty, "gpt-5")).toStrictEqual({ claude: {} });
});
});
describe("savedRouterWeights", () => {
it("should drop groups with no weights so they are not saved", () => {
expect(savedRouterWeights({ "gpt-5": { a: 0, b: 2 }, claude: {} })).toStrictEqual({ "gpt-5": { a: 0, b: 2 } });
});
it("should save no weights as null so the field is cleared", () => {
expect(savedRouterWeights({})).toBeNull();
expect(savedRouterWeights({ claude: {} })).toBeNull();
});
});
describe("hasNoPositiveWeight", () => {
it("should flag a group whose weights are all zero, as the proxy rejects it", () => {
expect(hasNoPositiveWeight({ a: 0, b: 0 })).toBe(true);
expect(hasNoPositiveWeight({ a: 0, b: 0.5 })).toBe(false);
expect(hasNoPositiveWeight({})).toBe(false);
});
});
describe("weightRows", () => {
it("should list the group's deployments with each one's share of traffic", () => {
const rows = weightRows(
"gpt-5",
{ a: 9, b: 1 },
[
deployment("a", "gpt-5", "openai/gpt-5"),
deployment("b", "gpt-5", "azure/gpt-5"),
deployment("c", "gpt-5", "bedrock/gpt-5"),
],
null,
);
expect(rows).toStrictEqual([
{ deploymentId: "a", label: "openai/gpt-5", weight: 9, sharePercent: 90 },
{ deploymentId: "b", label: "azure/gpt-5", weight: 1, sharePercent: 10 },
{ deploymentId: "c", label: "bedrock/gpt-5", weight: null, sharePercent: 0 },
]);
});
it("should match team deployments by their public model name, not their internal routing name", () => {
const rows = weightRows(
"gpt-5",
{},
[
deployment("team-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }),
deployment("other-group", "gpt-5-mini", "openai/gpt-5-mini"),
],
"team-1",
);
expect(rows.map((row) => row.deploymentId)).toStrictEqual(["team-dep"]);
});
it("should group a deployment with no team by its model name even if it carries a public name", () => {
const orphanedPublicName = {
model_name: "gpt-5-internal",
litellm_params: { model: "openai/gpt-5" },
model_info: { id: "x", team_public_model_name: "gpt-5" },
};
expect(weightRows("gpt-5", {}, [orphanedPublicName], null)).toStrictEqual([]);
expect(weightRows("gpt-5-internal", {}, [orphanedPublicName], null)).toHaveLength(1);
});
it("should leave out another team's deployments, which the proxy rejects", () => {
const deployments = [
deployment("global", "gpt-5", "openai/gpt-5"),
deployment("team-1-dep", "model_name_team-1_abc", "openai/gpt-5", { id: "team-1", publicName: "gpt-5" }),
deployment("team-2-dep", "model_name_team-2_def", "openai/gpt-5", { id: "team-2", publicName: "gpt-5" }),
];
expect(weightRows("gpt-5", {}, deployments, "team-1").map((row) => row.deploymentId)).toStrictEqual([
"global",
"team-1-dep",
]);
expect(weightRows("gpt-5", {}, deployments, null).map((row) => row.deploymentId)).toStrictEqual(["global"]);
});
it("should show no share until some weight is set", () => {
const rows = weightRows("gpt-5", {}, [deployment("a", "gpt-5", "openai/gpt-5")], null);
expect(rows[0].sharePercent).toBeNull();
});
it("should keep a weighted id whose deployment is gone so it can still be cleared", () => {
const rows = weightRows("gpt-5", { a: 1, deleted: 1 }, [deployment("a", "gpt-5", "openai/gpt-5")], null);
const staleRow = { deploymentId: "deleted", label: null, weight: 1, sharePercent: 50 };
expect(rows[1]).toStrictEqual(staleRow);
});
});
describe("parseWeightInput", () => {
it("should accept non-negative numbers, clear on empty and reject the rest", () => {
expect(parseWeightInput("2.5")).toBe(2.5);
expect(parseWeightInput("0")).toBe(0);
expect(parseWeightInput(" ")).toBeNull();
expect(parseWeightInput("-1")).toBeUndefined();
expect(parseWeightInput("abc")).toBeUndefined();
expect(parseWeightInput("1.")).toBe(1);
expect(parseWeightInput(".")).toBeUndefined();
expect(parseWeightInput("1e3")).toBeUndefined();
expect(parseWeightInput("0x10")).toBeUndefined();
});
});

105
ui/litellm-dashboard/src/components/common_components/routerWeights.ts Normal file
View file

@ -0,0 +1,105 @@
export type GroupWeights = Readonly<Record<string, number>>;
export type RouterWeights = Readonly<Record<string, GroupWeights>>;
export interface WeightableDeployment {
model_name?: string | null;
litellm_params?: { model?: string | null } | null;
model_info?: { id?: string | null; team_id?: string | null; team_public_model_name?: string | null } | null;
}
export interface WeightRow {
deploymentId: string;
label: string | null;
weight: number | null;
sharePercent: number | null;
}
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const isWeight = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value) && value >= 0;
export const routerWeightsFrom = (stored: unknown): RouterWeights =>
isRecord(stored)
? Object.fromEntries(
Object.entries(stored)
.filter((entry): entry is [string, Record<string, unknown>] => isRecord(entry[1]))
.map(([group, weights]) => [
group,
Object.fromEntries(
Object.entries(weights).filter((entry): entry is [string, number] => isWeight(entry[1])),
),
]),
)
: {};
const deploymentGroup = (deployment: WeightableDeployment): string | null =>
(deployment.model_info?.team_id ? deployment.model_info.team_public_model_name : null) ||
deployment.model_name ||
null;
const isWeightable = (deployment: WeightableDeployment, group: string, teamId: string | null): boolean => {
const ownerTeamId = deployment.model_info?.team_id ?? null;
const visibleToCaller = ownerTeamId === null || ownerTeamId === teamId;
return deploymentGroup(deployment) === group && Boolean(deployment.model_info?.id) && visibleToCaller;
};
export const withGroup = (weights: RouterWeights, group: string): RouterWeights =>
group in weights ? weights : { ...weights, [group]: {} };
export const withoutGroup = (weights: RouterWeights, group: string): RouterWeights =>
Object.fromEntries(Object.entries(weights).filter(([name]) => name !== group));
export const withDeploymentWeight = (
weights: RouterWeights,
group: string,
deploymentId: string,
weight: number | null,
): RouterWeights => {
const others = Object.fromEntries(Object.entries(weights[group] ?? {}).filter(([id]) => id !== deploymentId));
return { ...weights, [group]: weight === null ? others : { ...others, [deploymentId]: weight } };
};
export const savedRouterWeights = (weights: RouterWeights): RouterWeights | null => {
const nonEmpty = Object.entries(weights).filter(([, group]) => Object.keys(group).length > 0);
return nonEmpty.length > 0 ? Object.fromEntries(nonEmpty) : null;
};
export const hasNoPositiveWeight = (group: GroupWeights): boolean =>
Object.keys(group).length > 0 && !Object.values(group).some((weight) => weight > 0);
export const weightRows = (
group: string,
groupWeights: GroupWeights,
deployments: readonly WeightableDeployment[],
teamId: string | null,
): WeightRow[] => {
const total = Object.values(groupWeights).reduce((sum, weight) => sum + weight, 0);
const row = (deploymentId: string, label: string | null): WeightRow => {
const weight = groupWeights[deploymentId] ?? null;
return {
deploymentId,
label,
weight,
sharePercent: total > 0 ? Math.round(((weight ?? 0) / total) * 100) : null,
};
};
const current = deployments
.filter((deployment) => isWeightable(deployment, group, teamId))
.map((deployment) => row(deployment.model_info!.id!, deployment.litellm_params?.model ?? null));
const currentIds = new Set(current.map(({ deploymentId }) => deploymentId));
const stale = Object.keys(groupWeights)
.filter((deploymentId) => !currentIds.has(deploymentId))
.map((deploymentId) => row(deploymentId, null));
return [...current, ...stale];
};
const WEIGHT_DRAFT = /^(\d+\.?\d*|\.\d+)?$/;
export const parseWeightInput = (raw: string): number | null | undefined => {
const trimmed = raw.trim();
if (!WEIGHT_DRAFT.test(trimmed)) return undefined;
if (trimmed === "") return null;
const weight = Number(trimmed);
return isWeight(weight) ? weight : undefined;
};

1
ui/litellm-dashboard/src/components/organisms/create_key_button.integration.test.tsx
View file

@ -173,6 +173,7 @@ const ROUTER_SETTINGS_DEFAULT = {
model_group_alias: null,
enable_tag_filtering: false,
routing_strategy_args: null,
weights: null,
};
const SECTION_PAYLOAD_ADDITIONS: Record<keyof typeof SECTIONS, Record<string, unknown>> = {

1
ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
View file

@ -1784,6 +1784,7 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
key={routerSettingsKey}
ref={routerSettingsRef}
accessToken={accessToken || ""}
teamId={selectedCreateKeyTeam?.team_id}
value={routerSettings || undefined}
onChange={setRouterSettings}
modelData={

2
ui/litellm-dashboard/src/components/team/TeamInfo.tsx
View file

@ -114,6 +114,7 @@ import SearchToolSelector from "../search_tools/SearchToolSelector";
import SkillSelector from "../skills/SkillSelector";
import EditLoggingSettings from "./EditLoggingSettings";
import RouterSettingsAccordion, { RouterSettingsAccordionRef } from "../common_components/RouterSettingsAccordion";
import { RouterWeightsSummary } from "../common_components/RouterSettingsSummary";
import { StorePromptsToggle } from "../common_components/StorePromptsToggle";
import MemberModal from "./EditMembership";
import MemberPermissions from "./member_permissions";
@ -2673,6 +2674,7 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
{info.router_settings.enable_tag_filtering && (
<div>{t("teams.teamInfo.settingsView.router.tagFilteringEnabled")}</div>
)}
<RouterWeightsSummary weights={info.router_settings.weights} />
</div>
) : (
<div className="text-muted-foreground">{t("teams.teamInfo.settingsView.router.none")}</div>

16
ui/litellm-dashboard/src/i18n/locales/en.json
View file

@ -20380,5 +20380,21 @@
"query": "query",
"request": "request"
}
},
"routerWeights": {
"title": "Deployment Weights",
"description": "Split a model group's traffic across its deployments. A deployment with no weight gets no traffic once any weight in its group is set. Weights only apply with the simple-shuffle routing strategy.",
"ignoredForStrategy": "Weights are ignored with {{strategy}}. Select simple-shuffle to use them.",
"addGroup": "Add a model group",
"addGroupPlaceholder": "Select a model group to weight",
"noModels": "No model groups found",
"noDeployments": "No deployments found for this model group",
"unknownDeployment": "Deployment no longer exists",
"weightFor": "Weight for {{deployment}}",
"share": "{{percent}}% of traffic",
"noPositiveWeight": "Set at least one weight above 0, or remove this model group.",
"removeGroup": "Remove weights for {{group}}",
"summary": "Deployment Weights:",
"loadFailed": "Could not load this model group's deployments. Saved weights are kept."
}
}