Merge remote-tracking branch 'origin/main' into litellm_claude_code_marketplace_auth_required

This commit is contained in:
mateo-berri 2026-10-28 18:08:41 -07:00
commit f155703e59
93 changed files with 2390 additions and 829 deletions

1
.github/CODEOWNERS vendored
View file

@ -2,3 +2,4 @@
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai
/.github/workflows/zizmor.yml @yuneng-berri

View file

@ -1,7 +1,7 @@
name: Cost map guard
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
pull_request_target:
on:
pull_request:
branches:
- main
- "litellm_**"
@ -19,6 +19,7 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.base.sha }}
persist-credentials: false
- name: Fetch the pull request head and its merge base
id: revisions

View file

@ -1,7 +1,7 @@
name: Spend logs guard
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
pull_request_target:
on:
pull_request:
branches:
- main
@ -19,7 +19,7 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.base.ref }}
ref: ${{ github.event.pull_request.base.sha }}
persist-credentials: false
- name: Fetch the pull request head, its merge base and its reviews
id: revisions

View file

@ -561,6 +561,9 @@ jobs:
- name: router_code_coverage
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
- name: router_surface_check
run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
- name: test_chat_completion_imports
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py

View file

@ -27,10 +27,15 @@ jobs:
with:
persist-credentials: false
- name: Install zizmor
run: |
curl -fsSL --retry 3 -o "$RUNNER_TEMP/zizmor.tar.gz" \
https://github.com/zizmorcore/zizmor/releases/download/v1.31.1/zizmor-x86_64-unknown-linux-gnu.tar.gz
echo "d185f726bdaf92df78273217dfdd24ebcd6b0ef48cf3ce776ff5762b6e138c99 $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c -
tar xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor
- name: Run zizmor
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
with:
version: "1.24.1"
min-severity: medium
advanced-security: false
annotations: true
env:
GH_TOKEN: ${{ github.token }}
run: |
"$RUNNER_TEMP/zizmor" --no-ignores --no-config --min-severity=medium --format=github .

View file

@ -4387,12 +4387,17 @@ name = "litellm-router"
version = "0.1.0"
dependencies = [
"litellm-config",
"litellm-core-utils",
"litellm-inference",
"litellm-inference-messages",
"litellm-router-types",
"rstest",
]
[[package]]
name = "litellm-router-core"
version = "0.1.0"
[[package]]
name = "litellm-router-types"
version = "0.1.0"

View file

@ -1,13 +1,25 @@
pub fn parse_str_bool(value: &str) -> Option<bool> {
let token = value.trim_matches(|character: char| {
fn python_strip(value: &str) -> &str {
value.trim_matches(|character: char| {
character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}')
});
})
}
pub fn parse_str_bool(value: &str) -> Option<bool> {
let token = python_strip(value);
if token.eq_ignore_ascii_case("true") {
return Some(true);
}
token.eq_ignore_ascii_case("false").then_some(false)
}
pub fn parse_pydantic_bool(value: &str) -> Option<bool> {
match python_strip(value).to_ascii_lowercase().as_str() {
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
_ => None,
}
}
/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
pub fn parse_redis_bool(value: &str) -> bool {
value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
@ -34,4 +46,27 @@ mod tests {
) {
assert_eq!(parse_str_bool(input), expected, "{input:?}");
}
#[rstest]
#[case::yes(" YES ", Some(true))]
#[case::on("On", Some(true))]
#[case::one("1", Some(true))]
#[case::t("t", Some(true))]
#[case::y("y", Some(true))]
#[case::control_whitespace_true("\u{1c}TRUE\u{1f}", Some(true))]
#[case::no("No", Some(false))]
#[case::off("OFF", Some(false))]
#[case::zero("0", Some(false))]
#[case::f("F", Some(false))]
#[case::n("n", Some(false))]
#[case::false_word("false", Some(false))]
#[case::two("2", None)]
#[case::empty("", None)]
#[case::unknown("maybe", None)]
fn boolean_words_follow_pydantic_lax_bool_parsing(
#[case] input: &str,
#[case] expected: Option<bool>,
) {
assert_eq!(parse_pydantic_bool(input), expected, "{input:?}");
}
}

View file

@ -37,6 +37,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
.map(|value| value.expose().to_owned())
.or_else(|| std::env::var(name).ok())
};
let router = ModelRouter::from_config(config, &lookup);
let settings = &config.litellm_settings;
let http_settings = HttpSettings::from_layers([
HttpSettingsLayer::from_environment(&lookup),
@ -64,12 +65,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
Arc::new(EnvironmentSecrets::python_compatible(client)),
));
let resources = CoreResources::new(pool);
Ok(Arc::new(Gateway::new(
resources,
http,
secrets,
ModelRouter::from_model_list(&config.model_list),
)?))
Ok(Arc::new(Gateway::new(resources, http, secrets, router)?))
}
pub async fn build_redis(config: &Config) -> Result<RedisClients, Error> {

View file

@ -141,9 +141,13 @@ async fn provider_call(
&env_lookup,
)?;
let body = config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
let body = config.request_body(
config.transform_anthropic_messages_request(
trimmed,
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
)?,
&validated.headers,
shaping.settings.drop_params,
)?;
let environment = ValidatedEnvironment {
headers: config.request_headers(
@ -514,6 +518,84 @@ mod tests {
);
}
fn vertex_call_requesting(
shaping: MessagesShaping,
requested: &str,
drop_params: bool,
) -> MessagesCall {
MessagesCall {
body: body(
json!({"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
),
api_key: None,
api_base: Some("https://vertex.test".into()),
custom_llm_provider: Some("vertex_ai".into()),
litellm_params: Default::default(),
extra_headers: Some(Map::from_iter([
("authorization".to_string(), json!("Bearer vertex-token")),
("anthropic-version".to_string(), json!(requested)),
])),
provider_specific_header: None,
timeout: None,
shaping: MessagesShaping {
settings: MessagesSettings {
drop_params,
..shaping.settings
},
..shaping
},
}
}
fn vertex_project(name: &str) -> Option<String> {
(name == "VERTEXAI_PROJECT").then(|| "version-project".to_string())
}
#[rstest]
#[case::sdk_2("2026-10-21", false, "2026-10-21")]
#[case::sdk_1("2023-06-01", false, "vertex-2023-10-16")]
#[case::unknown_under_drop_params("latest", true, "vertex-2023-10-16")]
#[tokio::test]
async fn vertex_gets_the_requested_version_in_the_body_alone(
shaping: MessagesShaping,
#[case] requested: &str,
#[case] drop_params: bool,
#[case] body_version: &str,
) {
let call = provider_call_with_secrets(
vertex_call_requesting(shaping, requested, drop_params),
&vertex_project,
)
.await
.unwrap();
let version_headers: Vec<&str> = call
.wire
.headers
.iter()
.filter(|(name, _)| name.eq_ignore_ascii_case("anthropic-version"))
.map(|(_, value)| value.as_str())
.collect();
assert_eq!(
(version_headers, &call.wire.body["anthropic_version"]),
(Vec::<&str>::new(), &json!(body_version))
);
}
#[rstest]
#[tokio::test]
async fn vertex_refuses_an_unsupported_version_without_drop_params(shaping: MessagesShaping) {
assert!(matches!(
provider_call_with_secrets(
vertex_call_requesting(shaping, "latest", false),
&vertex_project
)
.await,
Err(Error::Llm(litellm_llms::Error::InvalidRequest(
litellm_llms::ErrorDetail::UnsupportedParameter { .. }
)))
));
}
#[rstest]
#[tokio::test]
async fn missing_credentials_are_reported_before_the_request_is_transformed(

View file

@ -77,6 +77,15 @@ pub trait BaseMessagesConfig: Sync {
headers
}
fn request_body(
&self,
request: MessagesRequest,
_headers: &Headers,
_drop_params: bool,
) -> Result<MessagesRequest, Error> {
Ok(request)
}
/// The JSON that goes on the wire, for a host whose body differs from the typed request:
/// Python's configs `pop("model")` when the model is addressed by the URL.
fn wire_body(&self, body: Value) -> Value {

View file

@ -1,6 +1,7 @@
# rules
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
# references

View file

@ -3,10 +3,10 @@ use litellm_auth_gcp::{
VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
get_vertex_ai_project_from_credentials, secret_names,
};
use litellm_http::request::has_header;
use litellm_http::request::{has_header, header_value, without_headers};
use litellm_llms_types::{
formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, VERSION_HEADER},
recognized::Recognized,
};
use litellm_router_types::LitellmParams;
@ -36,6 +36,8 @@ use litellm_llms_types::providers::vertex_ai::{
};
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
/// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic
/// payload addressed to a project and location, authenticated with a Google access token.
@ -90,12 +92,6 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
.params
.output_config
.and_then(|config| sanitize_output_config(config, accepts_effort)),
extra: request
.params
.extra
.into_iter()
.chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))])
.collect(),
..request.params
},
..request
@ -133,7 +129,35 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
}
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
merge_beta_headers(headers, vertex_feature_betas(request))
merge_beta_headers(
without_headers(headers, &[VERSION_HEADER]),
vertex_feature_betas(request),
)
}
fn request_body(
&self,
request: MessagesRequest,
headers: &Headers,
drop_params: bool,
) -> Result<MessagesRequest, Error> {
let version = vertex_version(
&request.model,
header_value(headers, VERSION_HEADER),
drop_params,
)?;
Ok(MessagesRequest {
params: MessagesOptionalParams {
extra: request
.params
.extra
.into_iter()
.chain([("anthropic_version".to_string(), json!(version))])
.collect(),
..request.params
},
..request
})
}
/// Vertex addresses the model in the URL and rejects it in the body.
@ -225,6 +249,26 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
)
}
fn vertex_version<'a>(
model: &str,
requested: Option<&'a str>,
drop_params: bool,
) -> Result<&'a str, Error> {
match requested.map(str::trim).unwrap_or_default() {
version if VERSIONS_SERVED_AS_VERTEX_DEFAULT.contains(&version) => Ok(ANTHROPIC_VERSION),
version if VERSIONS_VERTEX_ACCEPTS_IN_BODY.contains(&version) => Ok(version),
_ if drop_params => Ok(ANTHROPIC_VERSION),
version => Err(Error::InvalidRequest(
crate::ErrorDetail::UnsupportedParameter {
model: model.into(),
param: VERSION_HEADER.into(),
value: version.into(),
hint: String::new(),
},
)),
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
@ -471,6 +515,28 @@ mod tests {
);
}
#[rstest]
#[case::sdk_spelling("anthropic-version")]
#[case::any_case("Anthropic-Version")]
fn a_forwarded_anthropic_version_never_reaches_vertex(#[case] name: &str) {
let request: MessagesRequest = serde_json::from_value(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
}))
.unwrap();
assert_eq!(
VERTEX_ANTHROPIC_MESSAGES_CONFIG.request_headers(
vec![
(name.to_string(), "2026-10-21".to_string()),
("x-request-id".to_string(), "req-1".to_string()),
],
&request,
),
vec![("x-request-id".to_string(), "req-1".to_string())]
);
}
fn transformed(value: Value, context: &MessagesTransformContext) -> Value {
let request: MessagesRequest = serde_json::from_value(value).unwrap();
serde_json::to_value(
@ -482,16 +548,87 @@ mod tests {
}
#[rstest]
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
let body = transformed(
json!({
"model": "claude-sonnet-4-5@20250929",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}]
}),
&MessagesTransformContext::default(),
#[case::sdk_2(&[("anthropic-version", "2026-10-21")], false, "2026-10-21")]
#[case::any_case(&[("Anthropic-Version", "2026-10-21")], false, "2026-10-21")]
#[case::vertex_name(&[("anthropic-version", "vertex-2023-10-16")], false, "vertex-2023-10-16")]
#[case::sdk_1(&[("anthropic-version", "2023-06-01")], false, "vertex-2023-10-16")]
#[case::first_version(&[("anthropic-version", "2023-01-01")], false, "vertex-2023-10-16")]
#[case::blank(&[("anthropic-version", " ")], false, "vertex-2023-10-16")]
#[case::no_header(&[], false, "vertex-2023-10-16")]
#[case::unknown_under_drop_params(&[("anthropic-version", "latest")], true, "vertex-2023-10-16")]
fn the_requested_version_goes_in_the_body(
#[case] forwarded: &[(&str, &str)],
#[case] drop_params: bool,
#[case] body_version: &str,
) {
let request: MessagesRequest = serde_json::from_value(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
}))
.unwrap();
let headers: Headers = forwarded
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect();
let body = serde_json::to_value(
VERTEX_ANTHROPIC_MESSAGES_CONFIG
.request_body(request, &headers, drop_params)
.unwrap(),
)
.unwrap();
assert_eq!(body["anthropic_version"], json!(body_version));
}
#[rstest]
#[case::unreleased("2025-01-01")]
#[case::alias("latest")]
fn an_unsupported_version_is_refused_without_drop_params(#[case] requested: &str) {
let request: MessagesRequest = serde_json::from_value(json!({
"model": "claude-sonnet-4-5",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}],
}))
.unwrap();
let headers: Headers = vec![("anthropic-version".to_string(), requested.to_string())];
let Error::InvalidRequest(detail) = VERTEX_ANTHROPIC_MESSAGES_CONFIG
.request_body(request, &headers, false)
.unwrap_err()
else {
panic!("expected an invalid request");
};
assert_eq!(
detail.to_string(),
format!(
"claude-sonnet-4-5 does not support anthropic-version={requested}. \
To drop unsupported params, set `litellm.drop_params = True`."
)
);
assert_eq!(body["anthropic_version"], json!("vertex-2023-10-16"));
}
#[rstest]
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
let request: MessagesRequest = serde_json::from_value(json!({
"model": "claude-sonnet-4-5@20250929",
"max_tokens": 16,
"messages": [{"role": "user", "content": "hi"}]
}))
.unwrap();
let body = serde_json::to_value(
VERTEX_ANTHROPIC_MESSAGES_CONFIG
.request_body(
VERTEX_ANTHROPIC_MESSAGES_CONFIG
.transform_anthropic_messages_request(
request,
&MessagesTransformContext::default(),
)
.unwrap(),
&Vec::new(),
false,
)
.unwrap(),
)
.unwrap();
assert_eq!(body["model"], json!("claude-sonnet-4-5@20250929"));
let wire = VERTEX_ANTHROPIC_MESSAGES_CONFIG.wire_body(body);
assert_eq!(wire.get("model"), None);

View file

@ -1,3 +1,36 @@
//! The model catalog: per-model pricing, context limits and capabilities.
//!
//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
//!
//! # Structure
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
//! and keeps every entry in file order with its raw fields.
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
//! modalities and modes in `capabilities`).
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
//! failing the parse.
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
//! families. They are data in the file, not entries.
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
//! lost most of its models is refused), and `validate_registry` checks every entry against the
//! checked-in contract.
//!
//! # Lookups
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
//! prefixes are not stripped here; callers decide which keys to try.
//!
//! # Readers
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
//! performs I/O; loading and refreshing the file is the caller's job.
mod capabilities;
mod catalog;
mod error;

View file

@ -0,0 +1,9 @@
[package]
name = "litellm-router-core"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
[dependencies]

View file

@ -0,0 +1,63 @@
//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
//! and cooldowns around the attempts.
//!
//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
//! same interface. Until then it holds this overview and nothing else, so the interface is written
//! down in code before either side changes.
//!
//! # The public surface
//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
//! `call(op, request)`.
//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
//! `observe()`, `register_plugin()` and `retire()`.
//!
//! Every other member that outside code uses today has a named replacement among those. The old
//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
//! implements them over the entry points and warns on use. A CI check
//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
//! code touches; the list may only shrink.
//!
//! # Structure
//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
//! through its dynamic config.
//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
//! router never reads process globals. Static config (model list, strategy, retry, timeout and
//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
//! once per call.
//! - `Settings`: one typed, frozen view of the configuration a caller may read.
//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
//! see itself (a health check, an interceptor that marks a deployment failed).
//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
//! configured strategy, without calling it.
//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
//! a new registry version.
//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
//! policy is exhausted.
//!
//! # Execution
//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
//! started attempt is never replayed on a different executor.
//!
//! # Hosts
//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
//! serves them in the Rust server, where an operation without a Rust implementation makes the
//! config fail at build time, never partway through a request.
//!
//! # Plugins
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
//! the current request), never the router itself.

View file

@ -7,6 +7,7 @@ repository.workspace = true
[dependencies]
litellm-config.workspace = true
litellm-core-utils.workspace = true
litellm-inference.workspace = true
litellm-inference-messages.workspace = true
litellm-router-types.workspace = true

View file

@ -1,4 +1,4 @@
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_model_list(&config.model_list)` maps configured public names to provider deployments. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_config(&config, &env)` maps configured public names to provider deployments and reads `drop_params` from `litellm_settings` and each deployment, resolving an `os.environ/NAME` value through `env`. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
Lookup is exact and returns `None` for an unknown name. This extraction preserves the gateway's existing behavior: the last entry wins when public names repeat. Multiple deployments per model group, routing strategies, retries, cooldowns, and fallbacks are not implemented yet

View file

@ -2,16 +2,33 @@ mod deployment;
use std::collections::HashMap;
use litellm_config::Model;
use litellm_config::{Config, Spelled};
use litellm_core_utils::{serde_compat::parse_pydantic_bool, settings::Lookup};
use litellm_inference_messages::{MessagesSettings, MessagesShaping};
pub use deployment::Deployment;
#[derive(Clone, Debug, Default)]
pub struct Router(HashMap<String, Deployment>);
fn is_on(flag: Option<&Spelled<bool>>, env: &impl Lookup) -> bool {
match flag {
Some(Spelled::Value(on)) => *on,
Some(Spelled::Text(text)) => match text.strip_prefix("os.environ/") {
Some(name) => env
.get(name)
.is_some_and(|value| parse_pydantic_bool(&value) == Some(true)),
None => parse_pydantic_bool(text) == Some(true),
},
None => false,
}
}
impl Router {
pub fn from_model_list(model_list: &[Model]) -> Self {
model_list
pub fn from_config(config: &Config, env: &impl Lookup) -> Self {
let drops_params_globally = is_on(config.litellm_settings.drop_params.as_ref(), env);
config
.model_list
.iter()
.map(|model| {
(
@ -26,6 +43,14 @@ impl Router {
api_base: model.litellm_params.api_base.clone(),
custom_llm_provider: model.litellm_params.custom_llm_provider.clone(),
litellm_params: model.litellm_params.clone(),
shaping: MessagesShaping {
settings: MessagesSettings {
drop_params: drops_params_globally
|| is_on(model.litellm_params.drop_params.as_ref(), env),
..MessagesSettings::default()
},
..MessagesShaping::default()
},
..Deployment::default()
},
)
@ -43,3 +68,60 @@ impl FromIterator<(String, Deployment)> for Router {
Self(entries.into_iter().collect())
}
}
#[cfg(test)]
mod tests {
use litellm_config::Config;
use rstest::rstest;
use super::Router;
#[rstest]
#[case::unset("", "", false)]
#[case::deployment_flag("", "\n drop_params: true", true)]
#[case::deployment_word("", "\n drop_params: \" True \"", true)]
#[case::deployment_off("", "\n drop_params: false", false)]
#[case::deployment_unknown_word("", "\n drop_params: maybe", false)]
#[case::deployment_yes("", "\n drop_params: yes", true)]
#[case::deployment_no("", "\n drop_params: \"no\"", false)]
#[case::deployment_env_reference("", "\n drop_params: os.environ/DROP_PARAMS_FLAG", true)]
#[case::deployment_unset_env_reference(
"",
"\n drop_params: os.environ/MISSING_FLAG",
false
)]
#[case::global_env_reference(
"litellm_settings:\n drop_params: os.environ/DROP_PARAMS_FLAG\n",
"",
true
)]
#[case::global_flag("litellm_settings:\n drop_params: true\n", "", true)]
#[case::global_word("litellm_settings:\n drop_params: \"true\"\n", "", true)]
#[case::global_off("litellm_settings:\n drop_params: false\n", "", false)]
#[case::global_over_deployment_off(
"litellm_settings:\n drop_params: true\n",
"\n drop_params: false",
true
)]
fn configured_drop_params_reach_the_deployment(
#[case] settings: &str,
#[case] deployment: &str,
#[case] expected: bool,
) {
let config = Config::from_yaml(&format!(
"{settings}model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model{deployment}"
))
.unwrap();
assert_eq!(
Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG")
.then(|| " Yes ".to_owned()))
.get("public-model")
.unwrap()
.shaping
.settings
.drop_params,
expected
);
}
}

View file

@ -16,7 +16,7 @@ fn configuration_preserves_deployment_parameters(#[case] parameters: &str) {
"model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model\n {parameters}"
))
.unwrap();
let router = Router::from_model_list(&config.model_list);
let router = Router::from_config(&config, &|_: &str| None);
let deployment = router.get(&config.model_list[0].model_name).unwrap();
let params = &config.model_list[0].litellm_params;
@ -44,7 +44,7 @@ fn configuration_types_the_provider_params_of_a_deployment() {
azure_ad_token: ignored-by-every-group",
)
.unwrap();
let router = Router::from_model_list(&config.model_list);
let router = Router::from_config(&config, &|_: &str| None);
let params = &router.get("public-model").unwrap().litellm_params;
assert_eq!(
@ -74,7 +74,7 @@ fn lookup_uses_public_names(#[case] name: &str, #[case] expected: Option<&str>)
model: provider/b",
)
.unwrap();
let router = Router::from_model_list(&config.model_list);
let router = Router::from_config(&config, &|_: &str| None);
assert_eq!(router.get(name).map(|entry| entry.model.as_str()), expected);
}
@ -84,7 +84,7 @@ fn empty_configuration_has_no_deployment() {
let config = Config::from_yaml("model_list: []").unwrap();
assert!(
Router::from_model_list(&config.model_list)
Router::from_config(&config, &|_: &str| None)
.get("")
.is_none()
);

View file

@ -1336,7 +1336,6 @@ all_embedding_models = (
)
####### IMAGE GENERATION MODELS ###################
openai_image_generation_models = ["dall-e-2", "dall-e-3"]
####### VIDEO GENERATION MODELS ###################
openai_video_generation_models = ["sora-2"]

View file

@ -764,7 +764,6 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
# Used for web search interception across providers
LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search"
DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2"
DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2"
DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)

View file

@ -1592,14 +1592,6 @@ def completion_cost(
for part in ws_tier_parts
)
if (
(call_type == "aimage_generation" or call_type == "image_generation")
and model is not None
and isinstance(model, str)
and len(model) == 0
and custom_llm_provider == "azure"
):
model = "dall-e-2" # for dall-e-2, azure expects an empty model name
# Handle Inputs to completion_cost
prompt_tokens = 0
prompt_characters: int | None = None
@ -1674,7 +1666,8 @@ def completion_cost(
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
)
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
model = candidate_model
try:
if verbose_logger.isEnabledFor(logging.DEBUG):
verbose_logger.debug("selected model name for cost calculation: %s", model)

View file

@ -16,7 +16,6 @@ import litellm
# client is imported from litellm as it's a decorator
from litellm import client
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.get_llm_provider_logic import (
@ -299,17 +298,14 @@ def image_generation(
if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
else None
)
dynamic_api_key: str | None = None
if model is not None or custom_llm_provider is not None:
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=chatgpt_params,
)
else:
model = "dall-e-2"
custom_llm_provider = "openai" # default to dall-e-2 on openai
if model is None and custom_llm_provider is None:
raise litellm.BadRequestError(message="model is required", model="", llm_provider="")
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
litellm_params=chatgpt_params,
)
model_response.hidden_params["model"] = model
openai_params: Final = [
"user",
@ -712,7 +708,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
@client
def image_variation(
image: FileTypes,
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
model: str | None = None,
n: int = 1,
response_format: Literal["url", "b64_json"] = "url",
size: str | None = None,
@ -728,7 +724,7 @@ def image_variation(
litellm_params: Final = get_litellm_params(**kwargs)
# get the custom llm provider
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
model=model or "",
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
api_base=litellm_params.get("api_base", None),
api_key=litellm_params.get("api_key", None),
@ -747,7 +743,7 @@ def image_variation(
response: ImageResponse | None = None
provider_config: Final = ProviderConfigManager.get_provider_model_info(
model=model or "", # openai defaults to dall-e-2
model=model,
provider=llm_provider,
)
@ -772,7 +768,7 @@ def image_variation(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
model=model or None,
image=image,
timeout=litellm_params.get("timeout", None),
custom_llm_provider=custom_llm_provider,
@ -790,7 +786,7 @@ def image_variation(
model_response=model_response,
api_key=api_key,
api_base=api_base,
model=model,
model=model or None,
image=image,
timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
custom_llm_provider=custom_llm_provider,
@ -901,7 +897,7 @@ def image_edit(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
model, custom_llm_provider, _, _ = get_llm_provider(
model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
model=model or "",
custom_llm_provider=custom_llm_provider,
)

View file

@ -437,7 +437,6 @@ def get_llm_provider(
model in litellm.open_ai_chat_completion_models
or "ft:gpt-3.5-turbo" in model
or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
or model in litellm.openai_image_generation_models
or model.startswith("gpt-image")
or model in litellm.openai_video_generation_models
):

View file

@ -21,8 +21,8 @@ from pydantic import TypeAdapter
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.batch_utils import InMemoryFile
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.router_utils.batch_utils import InMemoryFile
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,

View file

@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
from litellm.router_utils.baseline_request import (
from litellm.router_strategy.auto_router.baseline_request import (
BASELINE_PARAMETERS,
capture_baseline_parameters,
)

View file

@ -13,13 +13,12 @@ from openai import (
BadRequestError,
OpenAI,
)
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel
import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -34,7 +33,6 @@ from litellm.types.utils import (
from litellm.utils import (
CustomStreamWrapper,
convert_to_model_response_object,
modify_url,
)
from ...types.llms.openai import HttpxBinaryResponseContent
@ -59,8 +57,6 @@ from .image_generation import (
)
from .image_generation.http_utils import azure_deployment_image_generation_json_body
_DECODED_JSON: Final = TypeAdapter(object)
class AzureOpenAIAssistantsAPIConfig:
"""
@ -860,11 +856,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers: dict,
deployment_name: str | None = None,
) -> httpx.Response:
"""
Implemented for azure dall-e-2 image gen calls
Alternative to needing a custom transport implementation
"""
if client is None:
_params: Final = {}
if timeout is not None:
@ -881,95 +872,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
else:
async_handler = client
if (
"images/generations" in api_base
and api_version
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
"2023-06-01-preview",
"2023-07-01-preview",
"2023-08-01-preview",
"2023-09-01-preview",
"2023-10-01-preview",
]
): # CREATE + POLL for azure dall-e-2 calls
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
data.pop(
"model", None
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
response = await async_handler.post(
url=api_base,
data=json.dumps(data),
headers=headers,
)
if "operation-location" in response.headers:
operation_location_url: Final = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
# Reject polling URLs that don't share an origin with ``api_base``.
# Without this an upstream-controlled or attacker-controlled
# value would receive the operator's Azure API key in the
# request headers below. VERIA-51.
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = await async_handler.get(
url=operation_location_url,
headers=headers,
)
await response.aread()
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
start_time: Final = time.time()
if "status" not in response.json():
# Don't reflect the raw response body — when the polling
# URL points at an internal JSON API (cloud metadata
# service etc.) reflecting it here turns Blind SSRF into
# Full-Read SSRF. VERIA-51.
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
await asyncio.sleep(int(response.headers.get("retry-after") or 10))
response = await async_handler.get(
url=operation_location_url,
headers=headers,
)
await response.aread()
if response.json()["status"] == "failed":
error_data: Final = response.json()
# Preserve Azure error details (e.g. content_policy_violation,
# inner_error, content_filter_results) as structured body so
# exception_type() can route them correctly.
_error_body: Final = error_data.get("error", error_data)
_error_msg: Final = (
_error_body.get("message", "Image generation failed")
if isinstance(_error_body, dict)
else json.dumps(error_data)
)
raise AzureOpenAIError(
status_code=400,
message=_error_msg,
body=error_data,
)
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
return httpx.Response(
status_code=200,
headers=response.headers,
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
return await async_handler.post(
url=api_base,
@ -988,11 +890,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers: dict,
deployment_name: str | None = None,
) -> httpx.Response:
"""
Implemented for azure dall-e-2 image gen calls
Alternative to needing a custom transport implementation
"""
if client is None:
_params: Final = {}
if timeout is not None:
@ -1006,87 +903,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
else:
sync_handler = client
if (
"images/generations" in api_base
and api_version
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
"2023-06-01-preview",
"2023-07-01-preview",
"2023-08-01-preview",
"2023-09-01-preview",
"2023-10-01-preview",
]
): # CREATE + POLL for azure dall-e-2 calls
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
data.pop(
"model", None
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
response = sync_handler.post(
url=api_base,
data=json.dumps(data),
headers=headers,
)
if "operation-location" in response.headers:
operation_location_url: Final = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = sync_handler.get(
url=operation_location_url,
headers=headers,
)
response.read()
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
start_time: Final = time.time()
if "status" not in response.json():
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
time.sleep(int(response.headers.get("retry-after") or 10))
response = sync_handler.get(
url=operation_location_url,
headers=headers,
)
response.read()
if response.json()["status"] == "failed":
error_data: Final = response.json()
# Preserve Azure error details (e.g. content_policy_violation,
# inner_error, content_filter_results) as structured body so
# exception_type() can route them correctly.
_error_body: Final = error_data.get("error", error_data)
_error_msg: Final = (
_error_body.get("message", "Image generation failed")
if isinstance(_error_body, dict)
else json.dumps(error_data)
)
raise AzureOpenAIError(
status_code=400,
message=_error_msg,
body=error_data,
)
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
return httpx.Response(
status_code=200,
headers=response.headers,
content=json.dumps(result).encode("utf-8"),
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
)
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
return sync_handler.post(
url=api_base,
@ -1193,10 +1009,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
deployment_name=model,
)
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
if provider_config is not None:
return provider_config.transform_image_generation_response(
model=data.get("model", "dall-e-2"),
model=data.get("model", ""),
raw_response=httpx_response,
model_response=model_response or ImageResponse(),
logging_obj=logging_obj,
@ -1329,10 +1145,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
headers=request_headers,
deployment_name=model,
)
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):
return provider_config.transform_image_generation_response(
model=data.get("model", "dall-e-2"),
model=data.get("model", ""),
raw_response=httpx_response,
model_response=model_response or ImageResponse(),
logging_obj=logging_obj,

View file

@ -4,14 +4,10 @@ from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
from .gpt_transformation import AzureGPTImageGenerationConfig
from .http_utils import azure_deployment_image_generation_json_body
__all__ = [
"AzureDallE2ImageGenerationConfig",
"AzureDallE3ImageGenerationConfig",
"AzureGPTImageGenerationConfig",
"azure_deployment_image_generation_json_body",
]
@ -21,11 +17,7 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
model = model.lower()
model = model.replace("-", "")
model = model.replace("_", "")
if model == "" or "dalle2" in model: # empty model is dall-e-2
return AzureDallE2ImageGenerationConfig()
elif "dalle3" in model:
return AzureDallE3ImageGenerationConfig()
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
return AzureFoundryMAIImageGenerationConfig()
else:
verbose_logger.debug(

View file

@ -1,7 +0,0 @@
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
class AzureDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
"""
Azure dall-e-2 image generation config
"""

View file

@ -1,7 +0,0 @@
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
class AzureDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
"""
Azure dall-e-3 image generation config
"""

View file

@ -3,15 +3,11 @@ from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
from .flux_transformation import AzureFoundryFluxImageGenerationConfig
from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
from .mai_transformation import AzureFoundryMAIImageGenerationConfig
__all__ = [
"AzureFoundryDallE2ImageGenerationConfig",
"AzureFoundryDallE3ImageGenerationConfig",
"AzureFoundryFluxImageGenerationConfig",
"AzureFoundryGPTImageGenerationConfig",
"AzureFoundryMAIImageGenerationConfig",
@ -22,11 +18,7 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi
model = model.lower()
model = model.replace("-", "")
model = model.replace("_", "")
if model == "" or "dalle2" in model: # empty model is dall-e-2
return AzureFoundryDallE2ImageGenerationConfig()
elif "dalle3" in model:
return AzureFoundryDallE3ImageGenerationConfig()
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
return AzureFoundryMAIImageGenerationConfig()
elif "flux" in model:
return AzureFoundryFluxImageGenerationConfig()

View file

@ -1,7 +0,0 @@
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
class AzureFoundryDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
"""
Azure dall-e-2 image generation config
"""

View file

@ -1,7 +0,0 @@
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
class AzureFoundryDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
"""
Azure dall-e-3 image generation config
"""

View file

@ -561,11 +561,8 @@ class BaseLLMAIOHTTPHandler:
organization: str | None = None,
headers: dict | None = None,
) -> ImageResponse:
if model is None:
raise ValueError("model is required for non-openai image variations")
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
model=model, # openai defaults to dall-e-2
model=model or "",
provider=LlmProviders(custom_llm_provider),
)

View file

@ -1,31 +1,5 @@
from typing import Final
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from .dalle2_transformation import DallE2ImageEditConfig
from .transformation import OpenAIImageEditConfig
__all__ = [
"DallE2ImageEditConfig",
"OpenAIImageEditConfig",
"get_openai_image_edit_config",
]
def get_openai_image_edit_config(model: str) -> BaseImageEditConfig:
"""
Get the appropriate OpenAI image edit config based on the model.
Args:
model: The model name (e.g., "dall-e-2", "gpt-image-1")
Returns:
The appropriate config instance for the model
"""
model_normalized: Final = model.lower().replace("-", "").replace("_", "")
if model_normalized == "dalle2":
return DallE2ImageEditConfig()
else:
# Default to standard OpenAI config for gpt-image-1 and other models
return OpenAIImageEditConfig()

View file

@ -1,98 +0,0 @@
from io import BufferedReader
from typing import TYPE_CHECKING, Any, Final, cast
from httpx._types import RequestFiles
import litellm
from litellm.images.utils import ImageEditRequestUtils
from litellm.types.images.main import ImageEditRequestParams
from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
from .transformation import OpenAIImageEditConfig
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
LiteLLMLoggingObj = _LiteLLMLoggingObj
else:
LiteLLMLoggingObj = Any
class DallE2ImageEditConfig(OpenAIImageEditConfig):
"""
DALL-E-2 specific configuration for image edit API.
DALL-E-2 only supports editing a single image (not an array).
Uses "image" field name instead of "image[]".
"""
def transform_image_edit_request(
self,
model: str,
prompt: str | None,
image: FileTypes | None,
image_edit_optional_request_params: dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> tuple[dict, RequestFiles]:
"""
Transform image edit request for DALL-E-2.
DALL-E-2 only accepts a single image with field name "image" (not "image[]").
"""
request_params: Final = {
"model": model,
**image_edit_optional_request_params,
}
if image is not None:
request_params["image"] = image
if prompt is not None:
request_params["prompt"] = prompt
request: Final = ImageEditRequestParams(**request_params)
request_dict: Final = cast(dict, request)
#########################################################
# Separate images and masks as `files` and send other parameters as `data`
#########################################################
_image_list: Final = request_dict.get("image")
_mask = request_dict.get("mask")
data_without_files: Final = {k: v for k, v in request_dict.items() if k not in ["image", "mask"]}
files_list: Final[list[tuple[str, Any]]] = []
# Handle image parameter - DALL-E-2 only supports single image
if _image_list is not None:
image_list: Final = [_image_list] if not isinstance(_image_list, list) else _image_list
# Validate only one image is provided
if len(image_list) > 1:
raise litellm.BadRequestError(
message="DALL-E-2 only supports editing a single image. Please provide one image.",
model=model,
llm_provider="openai",
)
# Use "image" field name (singular) for DALL-E-2
for _image in image_list:
if _image is not None:
self._add_image_to_files(
files_list=files_list,
image=_image,
field_name="image",
)
# Handle mask parameter if provided
if _mask is not None:
# Handle case where mask can be a list (extract first mask)
if isinstance(_mask, list):
_mask = _mask[0] if _mask else None
if _mask is not None:
mask_content_type: Final[str] = ImageEditRequestUtils.get_image_content_type(_mask)
if isinstance(_mask, BufferedReader):
files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
else:
files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
return data_without_files, files_list

View file

@ -96,7 +96,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
"""
# Build request params, only including non-None values
request_params: Final = {
"model": model,
**({"model": model} if model else {}),
**image_edit_optional_request_params,
}
if image is not None:

View file

@ -1,9 +1,3 @@
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from .dall_e_2_transformation import DallE2ImageGenerationConfig
from .dall_e_3_transformation import DallE3ImageGenerationConfig
from .gpt_transformation import GPTImageGenerationConfig
from .guardrail_translation import (
OpenAIImageGenerationHandler,
@ -11,18 +5,7 @@ from .guardrail_translation import (
)
__all__ = [
"DallE2ImageGenerationConfig",
"DallE3ImageGenerationConfig",
"GPTImageGenerationConfig",
"OpenAIImageGenerationHandler",
"guardrail_translation_mappings",
]
def get_openai_image_generation_config(model: str) -> BaseImageGenerationConfig:
if model.startswith("dall-e-2") or model == "": # empty model is dall-e-2
return DallE2ImageGenerationConfig()
elif model.startswith("dall-e-3"):
return DallE3ImageGenerationConfig()
else:
return GPTImageGenerationConfig()

View file

@ -1,81 +0,0 @@
from typing import TYPE_CHECKING, Final
import httpx
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
"""
OpenAI dall-e-2 image generation config
"""
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
return ["n", "response_format", "quality", "size", "user"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params: Final = self.get_supported_openai_params(model)
for k in non_default_params:
if k not in optional_params:
if k in supported_params:
optional_params[k] = non_default_params[k]
elif drop_params:
pass
else:
raise ValueError(
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
)
return optional_params
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
response: Final = raw_response.json()
stringified_response: Final = response
## LOGGING
logging_obj.post_call(
input=request_data.get("prompt", ""),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=stringified_response,
)
image_response: Final[ImageResponse] = convert_to_model_response_object(
response_object=stringified_response,
model_response_object=model_response,
response_type="image_generation",
)
# set optional params
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
image_response.quality = optional_params.get("quality", "standard") # always standard for dall-e-2
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-2
return image_response

View file

@ -1,81 +0,0 @@
from typing import TYPE_CHECKING, Final
import httpx
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
"""
OpenAI dall-e-3 image generation config
"""
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
return ["n", "response_format", "quality", "size", "user", "style"]
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
model: str,
drop_params: bool,
) -> dict:
supported_params: Final = self.get_supported_openai_params(model)
for k in non_default_params:
if k not in optional_params:
if k in supported_params:
optional_params[k] = non_default_params[k]
elif drop_params:
pass
else:
raise ValueError(
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
)
return optional_params
def transform_image_generation_response(
self,
model: str,
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: "LiteLLMLoggingObj",
request_data: dict,
optional_params: dict,
litellm_params: dict,
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
response: Final = raw_response.json()
stringified_response: Final = response
## LOGGING
logging_obj.post_call(
input=request_data.get("prompt", ""),
api_key=api_key,
additional_args={"complete_input_dict": request_data},
original_response=stringified_response,
)
image_response: Final[ImageResponse] = convert_to_model_response_object(
response_object=stringified_response,
model_response_object=model_response,
response_type="image_generation",
)
# set optional params
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
image_response.quality = optional_params.get("quality", "hd") # always hd for dall-e-3
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-3
return image_response

View file

@ -151,7 +151,7 @@ class OpenAIImageVariationsHandler:
) -> ImageResponse:
try:
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
model=model or "", # openai defaults to dall-e-2
model=model or "",
provider=LlmProviders.OPENAI,
)

View file

@ -1,6 +1,8 @@
from collections.abc import Mapping, Sequence
from typing import Final
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
@ -17,6 +19,41 @@ from litellm.types.router import GenericLiteLLMParams
from ....vertex_llm_base import VertexBase
from ..output_params_utils import sanitize_vertex_anthropic_output_params
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
_VERSIONS_VERTEX_ACCEPTS_IN_BODY: Final = frozenset({_VERTEX_DEFAULT_VERSION, "2026-10-21"})
def _requested_version(headers: Mapping[str, object]) -> str:
requested: Final = next(
(value for name, value in headers.items() if name.lower() == _VERSION_HEADER_VERTEX_READS_FROM_BODY),
"",
)
return requested.strip() if isinstance(requested, str) else ""
def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool) -> str | UnsupportedParamsError:
requested: Final = _requested_version(headers)
if requested in _VERSIONS_SERVED_AS_VERTEX_DEFAULT:
return _VERTEX_DEFAULT_VERSION
if requested in _VERSIONS_VERTEX_ACCEPTS_IN_BODY:
return requested
if drop_params:
return _VERTEX_DEFAULT_VERSION
return UnsupportedParamsError(
message=(
f"{model} does not support {_VERSION_HEADER_VERTEX_READS_FROM_BODY}={requested}. "
"To drop unsupported params, set `litellm.drop_params = True`."
),
llm_provider="vertex_ai",
model=model,
)
def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers
return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY}
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
@property
@ -198,7 +235,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
messages: list[dict],
anthropic_messages_optional_request_params: dict,
litellm_params: GenericLiteLLMParams,
headers: dict,
headers: dict[str, object], # mutable-ok: mirrors the provider transform contract
) -> dict:
anthropic_messages_request: Final = super().transform_anthropic_messages_request(
model=model,
@ -212,10 +249,28 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
self._remove_scope_from_cache_control(anthropic_messages_request)
anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16"
version: Final = _vertex_version(
headers, model=model, drop_params=drops_unsupported_params(litellm_params.drop_params)
)
if isinstance(version, UnsupportedParamsError):
raise version
anthropic_messages_request["anthropic_version"] = version
anthropic_messages_request.pop("model", None) # do not pass model in request body to vertex ai
sanitize_vertex_anthropic_output_params(anthropic_messages_request, model)
return anthropic_messages_request
def sign_request(
self,
headers: Mapping[str, str],
optional_params: Mapping[str, object],
request_data: Mapping[str, object],
api_base: str,
api_key: str | None = None,
model: str | None = None,
stream: bool | None = None,
fake_stream: bool | None = None,
) -> tuple[dict[str, str], bytes | None]: # mutable-ok: the caller sends these headers
return _without_version_header(headers), None

View file

@ -139,8 +139,8 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
)
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
from litellm.types.extraction import ExtractionResponse
from litellm.types.guardrails import GuardrailEventHooks

View file

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:

View file

@ -27,7 +27,7 @@ from litellm.constants import (
HEALTH_CHECK_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependency,
classify_strategy_router_model,
strategy_router_dependencies,

View file

@ -37,8 +37,8 @@ from litellm.proxy.spend_tracking.savings import (
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
)
from litellm.router_strategy.auto_router.baseline_request import baseline_request
from litellm.router_strategy.complexity_router.context_compaction import compaction_applied
from litellm.router_utils.baseline_request import baseline_request
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import BaselineRouteStamp
from litellm.types.utils import CallTypes, ModelInfo, Usage

View file

@ -19,6 +19,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import Span
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions
@ -30,7 +31,6 @@ from litellm.proxy.hooks.model_budget_shared_windows import (
SharedWindows,
)
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload

View file

@ -3290,7 +3290,7 @@ def _per_user_credential_names_for_groups(
GITHUB_COPILOT_AUTH_TYPE_KEY,
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
)
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment
for group in model_groups:

View file

@ -60,16 +60,16 @@ from litellm.repositories.table_repositories import PrismaTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
from litellm.router_strategy.auto_router.auto_router_model_naming import (
StrategyRouterDependencyRole,
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_strategy.complexity_router import ComplexityRouter
from litellm.router_strategy.complexity_router.request_models import (
AutoRouterRoutingTestRequest,
RequestComplexityRouterConfig,
)
from litellm.router_utils.auto_router_model_naming import (
StrategyRouterDependencyRole,
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
@ -411,7 +411,7 @@ async def validate_complexity_router_config(
auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict)
)
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
validate_complexity_router_config_write,
)

View file

@ -131,6 +131,23 @@ from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import ModelTableRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
carries_complexity_router_settings,
count_capability_routers,
gated_capabilities_of,
is_complexity_router_model,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
)
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
tuning_quota_violation,
)
from litellm.router_strategy.complexity_router import (
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
ClassificationRubric,
@ -144,20 +161,6 @@ from litellm.router_strategy.complexity_router import (
normalize_classification_prompt,
)
from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write
from litellm.router_utils.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
carries_complexity_router_settings,
count_capability_routers,
gated_capabilities_of,
is_complexity_router_model,
validate_complexity_router_config_placement,
validate_complexity_router_config_write,
validate_strategy_router_model_write,
)
from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.llms.bedrock import AwsSessionTag
from litellm.types.proxy.management_endpoints.model_management_endpoints import (

View file

@ -7,14 +7,14 @@ from typing import Final
from pydantic import Json, TypeAdapter, ValidationError
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
capability_limit_violation,
classify_strategy_router_model,
count_capability_routers,
gated_capabilities_of,
)
from litellm.router_utils.auto_router_tuning_baseline import (
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
mutable_tuned_identities,
tuning_quota_violation,

View file

@ -34,13 +34,16 @@ from litellm.repositories.prisma_protocols import DatabaseClient
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import TeamMembershipRepository
from litellm.router import Router
from litellm.router_strategy.auto_router.auto_router_model_naming import (
classify_strategy_router_model,
strategy_router_dependencies,
)
from litellm.router_strategy.complexity_router.config import (
ComplexityRouterConfigWrite,
normalize_classifier_config_aliases,
resolve_complexity_router_config_write,
)
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.router import Deployment, updateDeployment

View file

@ -168,12 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import (
)
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
STRATEGY_ROUTER_PARAM_FIELDS,
capability_limit_violation,
@ -181,12 +176,17 @@ from litellm.router_utils.auto_router_model_naming import (
count_capability_routers,
validate_complexity_router_config_placement,
)
from litellm.router_utils.auto_router_tuning_baseline import (
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
TUNING_BASELINE_PARAM_NAME,
mutable_tuned_identities,
snapshot_tuning_baselines,
tuning_limit_violation,
)
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
get_hidden_params_dict,
)
from litellm.router_utils.common_utils import resolve_model_group_alias
from litellm.router_utils.routing_groups import parse_routing_groups
from litellm.types.caching import RedisPipelineIncrementOperation

View file

@ -7,6 +7,29 @@
#
# Thank you ! We ❤️ you! - Krrish & Ishaan
"""
litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments.
Public surface. The router is moving behind a narrow interface; code outside
the router should end up using a Router through two kinds of members only:
- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages,
the files, batches, assistants and vector store wrappers, ...), each a thin
wrapper around one call;
- entry points (being added) that replace direct access to internals: registry() for
deployment lookups, settings() for configuration reads, health() for
cooldowns, blocks and usage, select() to pick a deployment without calling
it, apply() for runtime changes, observe() for routing events,
register_plugin() for strategies and sub-routers, and retire() to release
the router.
Everything else on the class is internal. The members outside code uses today
are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI
check (router_surface_check.py next to it) fails on any new outside use, so the
list only shrinks while callers move to the entry points. The same interface
is documented for the Rust core in litellm-rust/crates/router-core.
"""
import asyncio
import contextlib
import copy
@ -85,6 +108,15 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.batch_utils import (
batch_create_routing,
deployment_supported_endpoints,
get_router_metadata_variable_name,
is_batch_retrieve_call_type,
replace_model_in_jsonl,
should_replace_model_in_jsonl,
without_deployments_that_cannot_batch,
)
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import (
coerce_token_limit,
@ -148,6 +180,15 @@ from litellm.llms.openai_like.model_info import (
get_openai_compatible_model_info,
)
from litellm.responses.bridge_check import keep_registered_mode
from litellm.router_strategy.auto_router.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
)
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.context_compaction import (
@ -187,24 +228,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
replace_complexity_router_headers,
response_total_token_count,
)
from litellm.router_utils.auto_router_model_naming import (
AUTO_ROUTER_MODEL_PREFIX,
STRATEGY_ROUTER_PARAM_FIELDS,
GatedAutoRouterCapability,
capability_limit_violation,
claimed_capabilities,
classify_strategy_router_model,
count_capability_routers,
)
from litellm.router_utils.batch_utils import (
batch_create_routing,
deployment_supported_endpoints,
get_router_metadata_variable_name,
is_batch_retrieve_call_type,
replace_model_in_jsonl,
should_replace_model_in_jsonl,
without_deployments_that_cannot_batch,
)
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
from litellm.router_utils.clientside_credential_handler import (
get_dynamic_litellm_params,
@ -16271,7 +16294,7 @@ class Router:
to the deployment that actually served the request. Every attempt therefore
writes or clears, never just writes.
"""
from litellm.router_utils.baseline_request import capture_baseline_parameters
from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters
from litellm.types.router import BaselineRouteStamp
phase_attributes(routing_decision_attributes(routing_decision))

View file

@ -36,12 +36,12 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
from litellm.router_strategy.tag_based_routing import get_tags_from_request_kwargs
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
from litellm.router_utils.cooldown_callbacks import (
get_prometheus_logger_from_callbacks,
)

View file

@ -10,7 +10,7 @@ from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.caching.redis_cache import log_redis_failure
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60

View file

@ -10,7 +10,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed

View file

@ -13,9 +13,9 @@ from litellm import ModelResponse, token_counter, verbose_logger
from litellm._internal_context import with_service_target
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import LiteLLMPydanticObjectBase

View file

@ -9,7 +9,7 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.utils import LiteLLMPydanticObjectBase
from litellm.utils import print_verbose

View file

@ -15,8 +15,8 @@ from litellm._internal_context import with_service_target
from litellm._logging import verbose_logger, verbose_router_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.router import RouterErrors
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
from litellm.utils import get_utc_datetime, print_verbose

View file

@ -13,6 +13,7 @@ from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_router_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
@ -22,7 +23,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
add_fallback_headers_to_response,
get_fallback_error_info,
)
from litellm.router_utils.batch_utils import get_router_metadata_variable_name
from litellm.router_utils.cooldown_handlers import (
_first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils
cast_exception_status_to_int,

View file

@ -7664,7 +7664,6 @@ def validate_environment(
model in litellm.open_ai_chat_completion_models
or model in litellm.open_ai_text_completion_models
or model in litellm.open_ai_embedding_models
or model in litellm.openai_image_generation_models
or model.startswith("gpt-image")
):
if "OPENAI_API_KEY" in os.environ:
@ -10311,11 +10310,9 @@ class ProviderConfigManager:
provider: LlmProviders,
) -> BaseImageGenerationConfig | None:
if LlmProviders.OPENAI == provider:
from litellm.llms.openai.image_generation import (
get_openai_image_generation_config,
)
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
return get_openai_image_generation_config(model)
return GPTImageGenerationConfig()
elif LlmProviders.AZURE == provider:
from litellm.llms.azure.image_generation import (
get_azure_image_generation_config,
@ -10546,9 +10543,9 @@ class ProviderConfigManager:
provider: LlmProviders,
) -> BaseImageEditConfig | None:
if LlmProviders.OPENAI == provider:
from litellm.llms.openai.image_edit import get_openai_image_edit_config
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
return get_openai_image_edit_config(model=model)
return OpenAIImageEditConfig()
elif LlmProviders.HOSTED_VLLM == provider:
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config

View file

@ -187,13 +187,13 @@ ignored_function_names = [
def main():
router_file = [
"./litellm/router.py",
"./litellm/router_utils/batch_utils.py",
"./litellm/litellm_core_utils/batch_utils.py",
"./litellm/router_utils/pattern_match_deployments.py",
]
# router_file = [
# "../../litellm/router.py",
# "../../litellm/router_utils/pattern_match_deployments.py",
# "../../litellm/router_utils/batch_utils.py",
# "../../litellm/litellm_core_utils/batch_utils.py",
# ] ## LOCAL TESTING
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere
# tests_dir = "../../tests/" # LOCAL TESTING

View file

@ -0,0 +1,211 @@
{
"_doc": "Router members and litellm.router_utils modules used outside the router. Shrink-only: see tests/code_coverage_tests/router_surface_check.py.",
"members_outside": [
"_get_client",
"_zero_cost_cache",
"a_add_message",
"aadapter_completion",
"aanthropic_messages",
"abatch_completion",
"abatch_completion_fastest_response",
"acancel_batch",
"acancel_fine_tuning_job",
"acompletion",
"acreate_assistants",
"acreate_batch",
"acreate_file",
"acreate_fine_tuning_job",
"acreate_thread",
"adaptive_routers",
"adecisions",
"adelete_assistant",
"aembedding",
"afile_content",
"afile_delete",
"aget_assistants",
"aget_messages",
"aget_responses",
"aget_thread",
"alist_batches",
"alist_fine_tuning_jobs",
"allm_passthrough_route",
"amoderation",
"aocr",
"are_all_deployments_blocked",
"arefresh_model_info",
"aretrieve_batch",
"aretrieve_fine_tuning_job",
"arm_routing_read_prefetch",
"arun_thread",
"async_get_available_deployment",
"async_get_fully_unhealthy_model_names",
"async_get_healthy_deployments",
"async_pre_routing_hook",
"auto_routers",
"avector_store_create",
"avector_store_search",
"background_health_check_model_groups",
"bind_to_request_logging",
"cache",
"cache_responses",
"cached_deployment_model_info",
"cached_model_group_info",
"completion",
"complexity_routers",
"config_deployments",
"content_policy_fallbacks",
"context_window_fallbacks",
"cooldown_time",
"default_deployment",
"default_litellm_params",
"delete_deployment",
"deployment_latency_map",
"deployment_names",
"deployment_usable_by_team",
"deployments_for_request",
"deployments_named",
"discard",
"embedding",
"enable_health_check_routing",
"fallbacks",
"generate_model_id",
"get_all_deployments",
"get_available_deployment_for_pass_through",
"get_available_guardrail",
"get_candidate_model_ids_for_route",
"get_configured_display_name",
"get_configured_mode",
"get_configured_service_tiers",
"get_configured_token_limits",
"get_credential_deployment",
"get_deployment",
"get_deployment_by_model_group_name",
"get_deployment_credentials",
"get_deployment_credentials_with_provider",
"get_deployment_model_info",
"get_discovered_model_info",
"get_fully_blocked_model_names",
"get_model_access_groups",
"get_model_from_alias",
"get_model_group_info",
"get_model_group_usage",
"get_model_ids",
"get_model_info",
"get_model_list",
"get_model_list_from_model_alias",
"get_model_list_from_routing_groups",
"get_model_list_of_routed_group",
"get_model_listing_info",
"get_model_names",
"get_remaining_model_group_usage",
"get_routable_deployments",
"get_routable_upstream_model",
"get_router_deployment_budget_limiter",
"get_routing_group",
"get_settings",
"get_valid_args",
"guardrail_list",
"has_model_id",
"health_check_ignore_transient_errors",
"health_state_cache",
"is_deployment_blocked",
"is_model_access_group_for_wildcard_route",
"is_recognized_model",
"kubernetes_pod_discovery",
"map_team_model",
"model_group_alias",
"model_list",
"model_name_to_deployment_indices",
"model_names",
"pattern_router",
"priority_header_update",
"prompt_caching_pin_cache",
"provider_budget_config",
"quality_routers",
"resolve_model_name_from_model_id",
"resolve_served_model_name",
"resolved_litellm_models",
"routable_model_group",
"router_general_settings",
"routing_groups",
"routing_plugins",
"schedule_acompletion",
"search_tools",
"serve_through_request_logging",
"should_include_deployment",
"split_request_priority",
"team_model_to_deployment_indices",
"team_pattern_routers",
"team_ptu_ceiling",
"team_public_model_names",
"team_public_model_names_of",
"update_redis_cache",
"update_settings",
"upsert_deployment",
"wildcard_route_deployments",
"wildcard_routed_models"
],
"members_router_family": [
"_count_pre_call_check_tokens",
"_pre_call_checks",
"_update_kwargs_before_fallbacks",
"acompletion",
"adecisions",
"aembedding",
"allowed_fails",
"allowed_fails_policy",
"aresponses",
"async_function_with_fallbacks",
"async_get_healthy_deployments",
"cache",
"cached_deployment_model_info",
"cooldown_cache",
"cooldown_time",
"default_max_parallel_requests",
"deployments_named",
"disable_cooldowns",
"embedding",
"enable_pre_call_checks",
"enable_tag_filtering",
"explicit_allowed_fails",
"fallback_access_check",
"fallback_budget_check",
"fallbacks",
"get_all_deployments",
"get_allowed_fails_from_policy",
"get_deployment",
"get_model_group",
"get_model_ids",
"get_model_info",
"get_model_list",
"get_router_model_info",
"log_retry",
"model_list",
"provider_default_deployment_ids",
"routing_group_has_alternatives",
"routing_strategy",
"search_tools",
"set_custom_routing_strategy",
"slack_alerting_logger",
"tag_filtering_match_any",
"tag_routing_prefix",
"team_model_has_alternatives"
],
"router_utils_imports_outside": [
"litellm.router_utils.access_windows",
"litellm.router_utils.add_retry_fallback_headers",
"litellm.router_utils.clientside_credential_handler",
"litellm.router_utils.common_utils",
"litellm.router_utils.cooldown_handlers",
"litellm.router_utils.deployment_pricing",
"litellm.router_utils.fallback_event_handlers",
"litellm.router_utils.get_retry_from_policy",
"litellm.router_utils.health_state_cache",
"litellm.router_utils.pattern_match_deployments",
"litellm.router_utils.ptu_shares",
"litellm.router_utils.reasoning_effort_capability",
"litellm.router_utils.router_callbacks.track_deployment_metrics",
"litellm.router_utils.routing_groups",
"litellm.router_utils.search_api_router"
]
}

View file

@ -0,0 +1,235 @@
"""
Fence for the Router's public surface.
Code outside the router may use only the Router members listed in
router_surface_allowlist.json, and may import only the litellm.router_utils
modules listed there. The lists hold what is used today and may only shrink:
- a member or module used outside the router that is not listed fails the
check (use an existing entry point instead, or discuss widening the surface);
- a listed member or module that nothing outside the router uses any more
also fails, so the entry is removed in the same change.
"Outside" means every non-test file under litellm/ and enterprise/ except
litellm/router.py and the router's own modules. Uses from the router family
(litellm/router_utils, litellm/router_strategy) are listed separately, because
those modules become private parts of the router core in a later step.
Member uses are found the same way the boundary survey found them: attribute
access on a receiver whose name looks like a router (llm_router,
litellm_router_instance, router, ...) or on the Router class, plus
getattr/setattr/hasattr with a constant name on such a receiver. Members
include the container file endpoints Router.__init__ registers by name.
Run: python tests/code_coverage_tests/router_surface_check.py [--write]
--write regenerates the allowlist from the current tree (for shrinking it).
"""
import ast
import json
import os
import re
import sys
from typing import Dict, List, Set
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.json")
ROUTER_FILES = {
"litellm/router.py",
}
ROUTER_CLASS_FILES = ["litellm/router.py"]
ROUTER_CLASS_NAMES = {"Router"}
SCAN_ROOTS = ["litellm", "enterprise"]
SKIP_DIRS = {
".git",
"node_modules",
".venv",
"__pycache__",
"litellm-rust",
"ui",
"docs",
"cookbook",
"tests",
"test",
}
RECEIVER = re.compile(
r"(^|[._])(llm_router|router|litellm_router|litellm_router_instance|router_instance"
r"|_router|proxy_router|llm_router_instance|router_obj|mock_router|Router)$"
)
# Router.__init__ registers these with setattr(self, name, ...), so they never appear as self.<name>.
CONTAINER_FILE_ENDPOINTS = "litellm/containers/endpoints.json"
DYNAMIC_FUNCS = {"getattr", "setattr", "hasattr"}
def _area(rel: str) -> str:
if rel.startswith(("litellm/router_utils/", "litellm/router_strategy/")):
return "router_family"
return "outside"
def _is_test(rel: str) -> bool:
name = os.path.basename(rel)
return "/tests/" in rel or name.startswith("test_") or name.endswith("_test.py")
def router_members() -> Set[str]:
members: Set[str] = set()
for rel in ROUTER_CLASS_FILES:
with open(os.path.join(ROOT, rel), encoding="utf-8") as f:
tree = ast.parse(f.read())
for cls in tree.body:
if not isinstance(cls, ast.ClassDef) or cls.name not in ROUTER_CLASS_NAMES:
continue
for node in cls.body:
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
members.add(node.name)
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
members.add(node.target.id)
elif isinstance(node, ast.Assign):
members.update(t.id for t in node.targets if isinstance(t, ast.Name))
for node in ast.walk(cls):
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
and isinstance(node.ctx, ast.Store)
):
members.add(node.attr)
with open(os.path.join(ROOT, CONTAINER_FILE_ENDPOINTS), encoding="utf-8") as f:
for endpoint in json.load(f)["endpoints"]:
members.update((endpoint["name"], endpoint["async_name"]))
return {m for m in members if not (m.startswith("__") and m.endswith("__"))}
def _python_files():
for top in SCAN_ROOTS:
base = os.path.join(ROOT, top)
for dirpath, dirnames, filenames in os.walk(base):
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
for fn in filenames:
if fn.endswith(".py"):
path = os.path.join(dirpath, fn)
yield os.path.relpath(path, ROOT).replace(os.sep, "/"), path
def scan() -> Dict[str, Dict[str, Set[str]]]:
"""Return {"members": {area: {member: files}}, "router_utils_imports": {module: files}}."""
members = router_members()
uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}}
imports: Dict[str, Set[str]] = {}
for rel, path in _python_files():
if rel in ROUTER_FILES or _is_test(rel):
continue
with open(path, encoding="utf-8") as f:
src = f.read()
has_router = "router" in src.lower()
if not has_router:
continue
try:
tree = ast.parse(src)
except SyntaxError:
continue
area = _area(rel)
for name in member_uses(tree, members):
uses[area].setdefault(name, set()).add(rel)
if area == "outside":
for node in ast.walk(tree):
for mod in _router_utils_modules(node):
imports.setdefault(mod, set()).add(rel)
return {"members": uses, "router_utils_imports": imports}
def member_uses(tree: ast.AST, members: Set[str]) -> Set[str]:
"""The Router members a module uses, on an instance or on the Router class itself."""
used: Set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Attribute) and node.attr in members:
if RECEIVER.search(ast.unparse(node.value)):
used.add(node.attr)
elif (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in DYNAMIC_FUNCS
and len(node.args) >= 2
and isinstance(node.args[1], ast.Constant)
and node.args[1].value in members
and RECEIVER.search(ast.unparse(node.args[0]))
):
used.add(node.args[1].value)
return used
def _router_utils_modules(node: ast.AST) -> List[str]:
if isinstance(node, ast.ImportFrom) and node.module:
if node.module == "litellm.router_utils":
return ["litellm.router_utils." + a.name for a in node.names]
if node.module.startswith("litellm.router_utils."):
return [node.module]
if isinstance(node, ast.Import):
return [a.name for a in node.names if a.name.startswith("litellm.router_utils.")]
return []
def build_allowlist(found) -> dict:
return {
"_doc": (
"Router members and litellm.router_utils modules used outside the router. "
"Shrink-only: see tests/code_coverage_tests/router_surface_check.py."
),
"members_outside": sorted(found["members"]["outside"]),
"members_router_family": sorted(found["members"]["router_family"]),
"router_utils_imports_outside": sorted(found["router_utils_imports"]),
}
def compare(allow: dict, found) -> List[str]:
errors: List[str] = []
sections = [
("members_outside", found["members"]["outside"], "Router member"),
("members_router_family", found["members"]["router_family"], "Router member (router family)"),
("router_utils_imports_outside", found["router_utils_imports"], "litellm.router_utils import"),
]
for key, used, what in sections:
listed = set(allow.get(key, []))
for name in sorted(set(used) - listed):
files = ", ".join(sorted(used[name]))
errors.append(
f"new {what} outside the router: {name!r} in {files}. The router surface may only "
f"shrink: use an entry point instead."
)
for name in sorted(listed - set(used)):
errors.append(
f"{what} {name!r} is listed in {os.path.basename(ALLOWLIST)} [{key}] but nothing uses it "
f"any more: remove it from the list."
)
return errors
def main(argv: List[str]) -> int:
found = scan()
if "--write" in argv:
with open(ALLOWLIST, "w", encoding="utf-8") as f:
json.dump(build_allowlist(found), f, indent=2)
f.write("\n")
print(f"wrote {ALLOWLIST}")
return 0
with open(ALLOWLIST, encoding="utf-8") as f:
allow = json.load(f)
errors = compare(allow, found)
for e in errors:
print(e)
if errors:
return 1
print(
f"router surface ok: {len(allow['members_outside'])} members outside, "
f"{len(allow['members_router_family'])} from the router family, "
f"{len(allow['router_utils_imports_outside'])} router_utils modules imported outside"
)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))

View file

@ -129,4 +129,4 @@ litellm/proxy/utils.py prefetch_config_params prisma param_name.in `param_names`
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma [scope.entity_id_field].in `list(scope.entity_ids)` 0
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma api_key.in `list(scope.api_keys)` 0
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma not.in `list(scope.exclude_entity_ids)` 0
litellm/router_utils/auto_router_model_naming.py <module> raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0
litellm/router_strategy/auto_router/auto_router_model_naming.py <module> raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0

View file

@ -26,6 +26,20 @@ _DECISIONS_PROBE_REPLY: Final = JsonResponse(
"usage": {"input_tokens": 10, "output_tokens": 1},
},
)
_EXTRACTION_PROBE_REPLY: Final = JsonResponse(
content_type="application/json",
body={
"result": {
"model": "clairic",
"answers": {"reachable": {"type": "text", "value": "health", "confidence": 0.9}},
"truncated": False,
"usage": {"input_tokens": 10, "output_tokens": 0},
},
"success": True,
"errors": [],
"messages": [],
},
)
_CONFIGURED_PROBE_REPLY: Final = JsonResponse(
content_type="application/json",
body={
@ -287,6 +301,29 @@ def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sen
]
def test_extraction_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_extraction_probe(
gateway: Gateway,
) -> None:
with gateway.scenario() as scenario:
handle: Final = register_scenario(f"health-extraction-{uuid.uuid4().hex[:12]}", _EXTRACTION_PROBE_REPLY)
scenario.cleanups.callback(delete_scenario, handle)
model: Final = scenario.model(
model="cloudflare/@cf/cloudflare/clairic", api_base=handle.api_base(), api_key="synthetic-cloudflare-key"
)
report: Final = _health_report(gateway, model)
assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
assert _probes_sent_to(gateway, handle) == [
(
f"/{handle.scenario_id}/ai/run/@cf/cloudflare/clairic",
{
"model": "clairic",
"state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"),
"fields": {"reachable": {"type": "text", "instructions": "Any word from the text"}},
},
)
]
@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None:
with gateway.scenario() as scenario:

View file

@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import dataclass
from hashlib import sha256
from itertools import chain
from itertools import chain, count
from pathlib import Path
from typing import Final, Literal, TypeVar
@ -50,6 +50,7 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat
directory: Final = tmp_path_factory.mktemp("mcp-llm-gateway")
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
config: Final = directory / "stored-responses.yaml"
(directory / f"{STREAM_BLOCK_MODULE}.py").write_text(STREAM_BLOCK_SOURCE)
config.write_text(
yaml.safe_dump(
{
@ -57,8 +58,17 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat
"general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True},
"litellm_settings": {
**object_value(stock["litellm_settings"]),
"cost_margin_config": {"hosted_vllm": {"fixed_amount": FIXED_MARGIN}},
"cost_margin_config": {
"hosted_vllm": {"fixed_amount": FIXED_MARGIN},
"deepseek": {"fixed_amount": FIXED_MARGIN},
},
},
"guardrails": [
{
"guardrail_name": STREAM_BLOCK,
"litellm_params": {"guardrail": f"{STREAM_BLOCK_MODULE}.BlockGuard", "mode": "post_call"},
}
],
}
)
)
@ -79,7 +89,7 @@ def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway
yield private
Surface = Literal["chat", "responses", "messages", "messages_bridge"]
Surface = Literal["chat", "responses", "messages", "messages_bridge", "responses_bridge"]
SURFACES: Final[tuple[Surface, ...]] = ("chat", "responses", "messages", "messages_bridge")
ADD: Final = {"a": 2, "b": 3}
ANSWER: Final = "the sum is 5"
@ -88,6 +98,19 @@ AUTO: Final = {**GATEWAY_REF, "require_approval": "never"}
OUTAGE: Final = "bridge-outage"
DIRECT: Final = "answer-without-tools"
FIXED_MARGIN: Final = 0.25
STREAM_BLOCK: Final = "mcp-stream-block"
STREAM_BLOCK_MODULE: Final = "mcp_stream_block"
STREAM_BLOCK_DETAIL: Final = "blocked by output policy"
STREAM_BLOCK_SOURCE: Final = (
"from fastapi import HTTPException\n\n"
"from litellm.integrations.custom_guardrail import CustomGuardrail\n\n\n"
"class BlockGuard(CustomGuardrail):\n"
" async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):\n"
" async for _ in response:\n"
" pass\n"
f' raise HTTPException(status_code=400, detail="{STREAM_BLOCK_DETAIL}")\n'
" yield\n"
)
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
@ -128,6 +151,8 @@ class Billing:
first: tuple[int, int] = (10, 5)
follow_up: tuple[int, int] = (10, 5)
follow_up_fails: bool = False
follow_up_breaks: bool = False
tool_rounds: int = 1
EVEN_ROUNDS: Final = Billing()
@ -138,6 +163,10 @@ def _fixed_turn(tool: str) -> Callable[[Mapping[str, JsonValue]], Turn]:
return lambda _: Turn(tool, json.dumps(ADD), ANSWER)
def _slow_turn(tool: str, seconds: float) -> Callable[[Mapping[str, JsonValue]], Turn]:
return lambda _: Turn(tool, json.dumps({"seconds": seconds}), ANSWER)
def _echoing_turn(body: Mapping[str, JsonValue]) -> Turn:
names: Final = _tool_names(body)
return Turn(names[0] if names else "", json.dumps({"query": _prompt(body)}), _tool_result_text(body) or "")
@ -183,6 +212,50 @@ def _item_events(index: int, item: Mapping[str, JsonValue]) -> tuple[dict[str, J
)
def _event_stream(events: Sequence[Mapping[str, JsonValue]]) -> Reply:
return Reply(
content_type="text/event-stream",
chunks=tuple(
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()
for number, event in enumerate(events)
),
)
def _broken_follow_up(target: str, identity: str, tokens: tuple[int, int]) -> Reply:
outage: Final = {"message": FOLLOW_UP_OUTAGE, "type": "server_error", "code": None}
if target.endswith("/chat/completions"):
delta: Final = {"role": "assistant", "content": "the sum"}
started: Final = {
"id": f"chatcmpl-{identity}",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": delta, "finish_reason": None}],
}
return Reply(
content_type="text/event-stream",
chunks=tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in (started, {"error": outage})),
)
input_tokens, output_tokens = tokens
failed: Final[dict[str, JsonValue]] = {
"id": f"resp_{identity}",
"object": "response",
"created_at": 1,
"status": "failed",
"model": "gpt-4o-mini",
"output": [],
"error": {"code": "server_error", "message": FOLLOW_UP_OUTAGE},
"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": sum(tokens)},
}
return _event_stream(
(
{"type": "response.created", "response": {**failed, "status": "in_progress", "error": None, "usage": None}},
{"type": "response.failed", "response": failed},
)
)
def _responses_stream(response: Mapping[str, JsonValue], items: Sequence[Mapping[str, JsonValue]]) -> Reply:
item_events: Final = tuple(chain.from_iterable(_item_events(index, item) for index, item in enumerate(items)))
events: Final = (
@ -191,13 +264,7 @@ def _responses_stream(response: Mapping[str, JsonValue], items: Sequence[Mapping
*item_events,
{"type": "response.completed", "response": response},
)
return Reply(
content_type="text/event-stream",
chunks=tuple(
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()
for number, event in enumerate(events)
),
)
return _event_stream(events)
def _block_ids(message: JsonValue, kind: str, field: str) -> frozenset[str]:
@ -252,6 +319,8 @@ def _anthropic_tool_result_rule(body: Mapping[str, JsonValue]) -> Reply | None:
def _model_double(
plan: Callable[[Mapping[str, JsonValue]], Turn], billing: Billing = EVEN_ROUNDS, parallel: int = 1
) -> Callable[[Request], Reply]:
served: Final = count()
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target.endswith("/models"):
return _json({"object": "list", "data": []})
@ -260,11 +329,12 @@ def _model_double(
outage: Final = {"error": {"message": "scripted provider outage", "type": "server_error", "code": None}}
return Reply(status=500, body=json.dumps(outage).encode())
turn: Final = plan(body)
call_number: Final = next(served)
answered: Final = _has_tool_result(body)
if answered and billing.follow_up_fails:
failure: Final = {"error": {"message": FOLLOW_UP_OUTAGE, "type": "server_error", "code": None}}
return Reply(status=500, body=json.dumps(failure).encode())
done: Final = answered or DIRECT in _prompt(body)
done: Final = DIRECT in _prompt(body) or (answered and call_number >= billing.tool_rounds)
calls: Final = range(1, parallel + 1)
prompt_tokens, completion_tokens = billing.follow_up if answered else billing.first
identity: Final = uuid.uuid4().hex[:12]
@ -273,9 +343,15 @@ def _model_double(
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
}
if answered and billing.follow_up_breaks:
return _broken_follow_up(request.target, identity, (prompt_tokens, completion_tokens))
if request.target.endswith("/chat/completions"):
requested: Final = tuple(
{"id": f"call_{call}", "type": "function", "function": {"name": turn.tool, "arguments": turn.arguments}}
{
"id": f"call_{identity}_{call}",
"type": "function",
"function": {"name": turn.tool, "arguments": turn.arguments},
}
for call in calls
)
message: Final = (
@ -460,6 +536,7 @@ def _upstream_model(surface: Surface) -> str:
"responses": "openai/responses/gpt-4o-mini",
"messages": "anthropic/claude-sonnet-4-5",
"messages_bridge": "hosted_vllm/gpt-4o-mini",
"responses_bridge": "deepseek/gpt-4o-mini",
}[surface]
@ -475,11 +552,15 @@ def _rig(
billing: Billing = EVEN_ROUNDS,
prices: Mapping[str, JsonValue] | None = None,
parallel: int = 1,
slow_tool_seconds: float | None = None,
) -> Iterator[Rig]:
alias: Final = "llm" + uuid.uuid4().hex[:8]
plan: Final = (
_fixed_turn(f"{alias}-add") if slow_tool_seconds is None else _slow_turn(f"{alias}-slow", slow_tool_seconds)
)
with (
mcp_peer() as peer,
wire_server(_model_double(_fixed_turn(f"{alias}-add"), billing, parallel)) as wire,
mcp_peer(rich=slow_tool_seconds is not None) as peer,
wire_server(_model_double(plan, billing, parallel)) as wire,
gateway.scenario() as scenario,
):
server_id: Final = register_mcp(scenario, peer, alias)
@ -1048,6 +1129,262 @@ def test_bridged_next_turn_replays_the_mcp_turn_answer(gateway: Gateway) -> None
assert ANSWER in _assistant_texts(opening), opening
StreamedUpstream = Literal["messages_bridge", "responses_bridge"]
STREAMED_UPSTREAMS: Final[tuple[StreamedUpstream, ...]] = ("messages_bridge", "responses_bridge")
STREAMED_IDS: Final = ("native", "bridged")
SLOW_TOOL_SECONDS: Final = 5.0
def _events(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]:
return tuple(
JSON_OBJECT.validate_json(line.removeprefix("data: "))
for line in response.text.splitlines()
if line.startswith("data: ") and line != "data: [DONE]"
)
def _completed(response: httpx.Response) -> dict[str, JsonValue]:
finished: Final = tuple(
object_value(event["response"]) for event in _events(response) if event["type"] == "response.completed"
)
assert len(finished) == 1, response.text
return finished[0]
def _model_rows(call_id: str, settled: Callable[[list[dict[str, JsonValue]]], bool]) -> list[dict[str, JsonValue]]:
return eventually(
lambda: read_rows(
"SELECT request_id, status, spend, prompt_tokens, completion_tokens "
'FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type NOT IN (%s, %s)',
(call_id, "call_mcp_tool", "list_mcp_tools"),
),
settled,
seconds=70,
return_last_on_timeout=True,
)
def _billed(rows: Sequence[Mapping[str, JsonValue]]) -> float:
return sum(float(str(row["spend"])) for row in rows)
def _bills_at_least(expected: float) -> Callable[[list[dict[str, JsonValue]]], bool]:
return lambda rows: bool(rows) and _billed(rows) >= expected - 1e-9
def _statuses(rows: Sequence[Mapping[str, JsonValue]]) -> list[str]:
return sorted(str(row["status"]) for row in rows)
def _logs_a_failure_billing(expected: float) -> Callable[[list[dict[str, JsonValue]]], bool]:
return lambda rows: "failure" in _statuses(rows) and _billed(rows) >= expected - 1e-9
def _limited_key(rig: Rig) -> str:
return rig.scenario.key(object_permission={"mcp_servers": [rig.server_id]}, max_parallel_requests=1)
def _answers_directly(rig: Rig, key: str) -> int:
return rig.send(key, [AUTO], prompt=f"{DIRECT} {rig.alias}").status_code
STREAMED_ROUNDS: Final = (
pytest.param("messages_bridge", 1, id="native"),
pytest.param("responses_bridge", 1, id="bridged"),
pytest.param("messages_bridge", 2, id="native-two-tool-rounds"),
pytest.param("responses_bridge", 2, id="bridged-two-tool-rounds"),
pytest.param("messages_bridge", 0, id="native-no-tool-call"),
pytest.param("responses_bridge", 0, id="bridged-no-tool-call"),
)
@pytest.mark.parametrize(("upstream", "tool_rounds"), STREAMED_ROUNDS)
def test_streaming_responses_auto_execute_bills_every_round_once_on_one_success_row(
gateway: Gateway, upstream: StreamedUpstream, tool_rounds: int
) -> None:
rounds: Final = (FIRST_ROUND, *(FOLLOW_UP_ROUND,) * tool_rounds)
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, tool_rounds=max(tool_rounds, 1))
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
prompt: Final = None if tool_rounds else f"{DIRECT} {rig.alias}"
response: Final = rig.send(_granted_key(rig), [AUTO], prompt=prompt, stream=True)
assert response.status_code == 200, response.text
completed: Final = _completed(response)
assert rig.final_text(completed) == ANSWER, response.text
assert len(rig.upstream_tools()) == len(rounds)
assert len(_peer_add_calls(rig.peer)) == tool_rounds
expected: Final = _price(*rounds) + FIXED_MARGIN
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _bills_at_least(expected))
assert [row["status"] for row in rows] == ["success"], rows
row: Final = rows[0]
assert (row["prompt_tokens"], row["completion_tokens"]) == (
sum(prompt_tokens for prompt_tokens, _ in rounds),
sum(completion_tokens for _, completion_tokens in rounds),
), row
usage: Final = object_value(completed["usage"])
assert (usage["input_tokens"], usage["output_tokens"]) == (row["prompt_tokens"], row["completion_tokens"])
assert float(str(row["spend"])) == pytest.approx(expected), row
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
def test_streaming_responses_rejected_follow_up_bills_round_one_once(
gateway: Gateway, upstream: StreamedUpstream
) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True)
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True)
assert FOLLOW_UP_OUTAGE in response.text, response.text
assert ANSWER not in response.text, response.text
assert len(rig.upstream_tools()) == 2
expected: Final = _price(FIRST_ROUND) + FIXED_MARGIN
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _bills_at_least(expected))
assert _billed(rows) == pytest.approx(expected), rows
BROKEN_FOLLOW_UPS: Final = (
pytest.param("messages_bridge", FOLLOW_UP_ROUND, id="native"),
pytest.param("responses_bridge", None, id="bridged"),
)
@pytest.mark.parametrize(("upstream", "reported_usage"), BROKEN_FOLLOW_UPS)
def test_streaming_responses_follow_up_broken_mid_stream_bills_round_one_once_and_itself_on_a_failure_row(
gateway: Gateway, upstream: StreamedUpstream, reported_usage: tuple[int, int] | None
) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_breaks=True)
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True)
assert ANSWER not in response.text, response.text
assert len(rig.upstream_tools()) == 2
round_one: Final = _price(FIRST_ROUND) + FIXED_MARGIN
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(round_one))
assert _statuses(rows) == ["failure", "success"], rows
(success,) = (row for row in rows if row["status"] == "success")
(failure,) = (row for row in rows if row["status"] == "failure")
assert (success["prompt_tokens"], success["completion_tokens"]) == FIRST_ROUND, rows
assert float(str(success["spend"])) == pytest.approx(round_one), rows
failed_round: Final = (int(str(failure["prompt_tokens"])), int(str(failure["completion_tokens"])))
assert min(failed_round) > 0, rows
assert reported_usage in (None, failed_round), rows
assert float(str(failure["spend"])) == pytest.approx(_price(failed_round)), rows
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_once(
gateway: Gateway, upstream: StreamedUpstream
) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
key: Final = _granted_key(rig)
one_round: Final = rig.send(key, [], prompt=f"{DIRECT} {rig.alias}", stream=True, guardrails=[STREAM_BLOCK])
assert STREAM_BLOCK_DETAIL in one_round.text, one_round.text
one_round_rows: Final = _model_rows(
one_round.headers["x-litellm-call-id"], _logs_a_failure_billing(_price(FIRST_ROUND) + FIXED_MARGIN)
)
rig.wire.drain()
response: Final = rig.send(key, [AUTO], stream=True, guardrails=[STREAM_BLOCK])
assert STREAM_BLOCK_DETAIL in response.text, response.text
assert len(rig.upstream_tools()) == 2 and len(_peer_add_calls(rig.peer)) == 1
expected: Final = _price(FIRST_ROUND, FOLLOW_UP_ROUND) + FIXED_MARGIN
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(expected))
assert _statuses(rows) == _statuses(one_round_rows), (rows, one_round_rows)
assert _billed(rows) == pytest.approx(expected), rows
@pytest.mark.parametrize(
("surface", "stream"),
(("responses", True), ("responses", False), ("chat", False)),
ids=("responses-stream", "responses", "chat"),
)
def test_auto_executed_tool_call_runs_inside_the_requests_one_parallel_slot(
gateway: Gateway, surface: Surface, stream: bool
) -> None:
with _rig(gateway, surface, slow_tool_seconds=SLOW_TOOL_SECONDS) as rig:
key: Final = _limited_key(rig)
with ThreadPoolExecutor(max_workers=1) as pool:
running: Final = pool.submit(rig.send, key, [AUTO], None, stream=stream)
eventually(lambda: _peer_add_calls(rig.peer), lambda found: len(found) == 1, seconds=30)
during: Final = _answers_directly(rig, key)
served: Final = running.result(timeout=90)
assert during == 429
assert served.status_code == 200 and ANSWER in served.text, served.text
eventually(lambda: _answers_directly(rig, key), lambda status: status == 200, seconds=30)
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
def test_streaming_responses_hung_up_during_the_tool_call_bills_each_served_round_once_and_frees_the_slot(
gateway: Gateway, upstream: StreamedUpstream
) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
with _rig(
gateway,
"responses",
upstream=upstream,
billing=billing,
prices=PRICES,
slow_tool_seconds=SLOW_TOOL_SECONDS,
) as rig:
key: Final = _limited_key(rig)
with rig.gateway.client.stream(
"POST",
"/v1/responses",
headers={"Authorization": f"Bearer {key}"},
json={"model": rig.model, "input": f"add {rig.alias}", "tools": [AUTO], "stream": True},
timeout=90,
) as hung_up:
assert hung_up.status_code == 200
call_id: Final = hung_up.headers["x-litellm-call-id"]
eventually(lambda: _peer_add_calls(rig.peer), lambda found: len(found) == 1, seconds=30)
eventually(lambda: _answers_directly(rig, key), lambda status: status == 200, seconds=60)
posted: Final = tuple(
JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST"
)
served: Final = tuple(
FOLLOW_UP_ROUND if _has_tool_result(body) else FIRST_ROUND for body in posted if DIRECT not in _prompt(body)
)
expected: Final = _price(*served) + FIXED_MARGIN
rows: Final = _model_rows(call_id, _bills_at_least(expected))
assert served and rows, (served, rows)
assert _billed(rows) == pytest.approx(expected), (served, rows)
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
def test_streaming_responses_repeated_from_the_response_cache_bills_nothing(
gateway: Gateway, upstream: StreamedUpstream
) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
key: Final = _granted_key(rig)
paid: Final = rig.send(key, [AUTO], stream=True)
assert paid.status_code == 200 and rig.final_text(_completed(paid)) == ANSWER, paid.text
paid_price: Final = _price(FIRST_ROUND, FOLLOW_UP_ROUND) + FIXED_MARGIN
_model_rows(paid.headers["x-litellm-call-id"], _bills_at_least(paid_price))
assert len(rig.upstream_tools()) == 2
cached: Final = rig.send(key, [AUTO], stream=True)
assert cached.status_code == 200 and rig.final_text(_completed(cached)) == ANSWER, cached.text
assert rig.upstream_tools() == ()
rows: Final = _model_rows(
cached.headers["x-litellm-call-id"], lambda found: "success" in {row["status"] for row in found}
)
assert "success" in {row["status"] for row in rows} and _billed(rows) == 0.0, rows
def test_bridged_next_turn_after_a_streamed_mcp_turn_replays_its_answer(gateway: Gateway) -> None:
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
with _rig(gateway, "responses", upstream="responses_bridge", billing=billing, prices=PRICES) as rig:
key: Final = _granted_key(rig)
first: Final = rig.send(key, [AUTO], stream=True)
assert first.status_code == 200, first.text
first_id: Final = string_value(_completed(first)["id"])
_model_rows(first.headers["x-litellm-call-id"], _bills_at_least(_price(FIRST_ROUND, FOLLOW_UP_ROUND)))
rig.wire.drain()
second: Final = rig.send(key, [AUTO], previous_response_id=first_id, cache={"no-cache": True})
assert second.status_code == 200, second.text
opening: Final = next(
JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST"
)
assert ANSWER in _assistant_texts(opening), opening
Bridge = Literal["chat", "responses", "messages"]
BRIDGES: Final[tuple[Bridge, ...]] = ("chat", "responses")
Client = Literal["sync", "async"]

View file

@ -0,0 +1,418 @@
import json
import math
import re
import uuid
from collections.abc import Iterator, Mapping
from dataclasses import dataclass
from pathlib import Path
from typing import Final, Literal
import httpx
import pytest
import yaml
from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
Route = Literal["/v1/extraction", "/extraction"]
_API_KEY: Final = "synthetic-cloudflare-key"
_CLAIRIC: Final = "cloudflare/@cf/cloudflare/clairic"
_CLAIRIC_PATH: Final = "/ai/run/@cf/cloudflare/clairic"
_INPUT_TOKENS: Final = 412
_EMAIL: Final = "billing@acmetools.example"
_SCENARIO: Final = re.compile(r"scenario-([a-z-]+)")
_JSON_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]])
_FIELDS: Final[dict[str, JsonValue]] = {
"vendor": {"type": "text", "instructions": "Who sent the invoice?"},
"total": {"type": "number", "unit": "USD"},
}
_SPEND_QUERY: Final = (
'SELECT spend, status, call_type, model_group, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" '
"WHERE request_id = %s"
)
_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120)
_REQUEST_GUARDRAIL: Final = "extraction-mask-email-in"
_RESPONSE_GUARDRAIL: Final = "extraction-mask-email-out"
_OWNED_MODEL: Final = "invoice-reader"
def _state(marker: str, scenario: str = "plain") -> str:
return f"Invoice from Acme Tools, total $174. scenario-{scenario} marker-{marker}"
def _vendor_span(state: str) -> list[JsonValue]:
start: Final = state.index("Acme Tools")
return [start, start + len("Acme Tools")]
def _email_span(state: str) -> list[JsonValue]:
start: Final = state.index(_EMAIL)
return [start, start + len(_EMAIL)]
def _answers(state: str) -> dict[str, JsonValue]:
return {
"vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)},
"total": {"type": "number", "value": 174, "confidence": 0.95},
}
def _envelope(result: Mapping[str, JsonValue]) -> Reply:
return Reply(body=json.dumps({"result": result, "success": True, "errors": [], "messages": []}).encode())
def _clairic_reply(request: Request) -> Reply:
state: Final = JSON_OBJECT.validate_json(request.body)["state"]
assert isinstance(state, str), request.body
return _envelope(
{
"model": "clairic",
"answers": _answers(state),
"truncated": False,
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
}
)
def _expected_response(model: str, state: str) -> dict[str, JsonValue]:
return {
"model": model,
"answers": _answers(state),
"truncated": False,
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
}
def _upstream_body(request: Request) -> dict[str, JsonValue]:
assert (request.method, request.target) == ("POST", _CLAIRIC_PATH), (request.method, request.target)
assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers
return JSON_OBJECT.validate_json(request.body)
def _body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]:
return {"model": model, "state": _state(marker), "fields": _FIELDS, **extra}
def _extract(
gateway: Gateway,
model: str,
marker: str,
*,
route: Route = "/v1/extraction",
key: str | None = None,
**extra: JsonValue,
) -> httpx.Response:
return gateway.request("POST", route, _body(model, marker, **extra), key=key)
def _number(value: JsonValue) -> float:
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
return float(value)
def _expected_spend() -> float:
cost_map: Final = JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes())
priced: Final = cost_map.get(_CLAIRIC)
assert priced is not None, f"{_CLAIRIC} has no cost map entry"
return _INPUT_TOKENS * _number(object_value(priced)["input_cost_per_token"])
def _spend_row(call_id: str) -> dict[str, JsonValue]:
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
return rows[0]
def _assert_billed(gateway: Gateway, response: httpx.Response, model: str) -> None:
expected: Final = _expected_spend()
assert math.isclose(float(response.headers["x-litellm-response-cost"]), expected, rel_tol=1e-9), response.headers
call_id: Final = response.headers["x-litellm-call-id"]
row: Final = _spend_row(call_id)
assert (row["status"], row["call_type"], row["model_group"], row["prompt_tokens"], row["completion_tokens"]) == (
"success",
"aextraction",
model,
_INPUT_TOKENS,
0,
), row
assert math.isclose(_number(row["spend"]), expected, rel_tol=1e-9), row
listed: Final = gateway.request("GET", "/spend/logs", params={"request_id": call_id})
assert listed.status_code == 200, listed.text
(reported,) = _JSON_ROWS.validate_json(listed.content)
assert (reported["request_id"], reported["model_group"]) == (call_id, model), reported
assert math.isclose(_number(reported["spend"]), expected, rel_tol=1e-9), reported
@pytest.mark.parametrize("route", ("/v1/extraction", "/extraction"))
def test_extraction_reaches_clairic_on_both_routes_and_is_billed(gateway: Gateway, route: Route) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
response: Final = _extract(gateway, model, marker, route=route, metadata={"tags": [f"audit-{marker}"]})
assert response.status_code == 200, response.text
assert response.json() == _expected_response(model, _state(marker))
(request,) = wire.drain()
assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS}
_assert_billed(gateway, response, model)
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
("unknown_type", {"fields": {"vendor": {"type": "paragraph"}}}),
("list_without_items", {"fields": {"items": {"type": "list"}}}),
("remote_image", {"images": ["https://images.example/receipt.png"]}),
)
@pytest.mark.parametrize(("label", "change"), _INVALID_BODIES, ids=[label for label, _ in _INVALID_BODIES])
def test_invalid_extraction_bodies_are_refused_before_any_upstream_call(
gateway: Gateway, label: str, change: dict[str, JsonValue]
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/extraction", {**_body(model, marker), **change})
assert response.status_code == 400, (label, response.text)
assert "Invalid Extraction request" in response.text, (label, response.text)
assert wire.drain() == (), label
@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("stream", True)), ids=("temperature", "stream"))
def test_unsupported_top_level_params_are_refused_unless_the_deployment_drops_params(
gateway: Gateway, param: str, value: JsonValue
) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
strict: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
dropping: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY, drop_params=True)
refused: Final = _extract(gateway, strict, marker, **{param: value})
assert refused.status_code == 400, refused.text
assert param in refused.text, refused.text
assert wire.drain() == ()
dropped: Final = _extract(gateway, dropping, marker, **{param: value})
assert dropped.status_code == 200, dropped.text
assert dropped.headers["content-type"].startswith("application/json"), dropped.headers
assert dropped.json() == _expected_response(dropping, _state(marker))
(request,) = wire.drain()
assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS}
_assert_billed(gateway, dropped, dropping)
def test_key_checks_on_extraction_match_chat(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
anonymous: Final = gateway.client.post("/v1/extraction", json=_body(model, marker))
assert anonymous.status_code == 401, anonymous.text
restricted: Final = scenario.key(models=[f"other-{uuid.uuid4().hex}"])
refused: Final = _extract(gateway, model, marker, key=restricted)
chat_refused: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": marker}]},
key=restricted,
)
assert 400 <= refused.status_code < 500, refused.text
assert refused.status_code == chat_refused.status_code, (refused.text, chat_refused.text)
assert wire.drain() == ()
allowed: Final = scenario.key(models=[model], max_budget=1e-06)
first: Final = _extract(gateway, model, marker, key=allowed)
assert first.status_code == 200, first.text
assert len(wire.drain()) == 1
blocked, upstream_calls = eventually(
lambda: (_extract(gateway, model, uuid.uuid4().hex, key=allowed), len(wire.drain())),
lambda answer: answer[0].status_code != 200,
seconds=70,
)
chat_blocked: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": marker}]},
key=allowed,
)
assert 400 <= blocked.status_code < 500, blocked.text
assert blocked.status_code == chat_blocked.status_code, (blocked.text, chat_blocked.text)
assert upstream_calls == 0
assert wire.drain() == ()
def test_a_small_key_tpm_limit_serves_extraction_and_counts_its_input_tokens(gateway: Gateway) -> None:
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
key: Final = scenario.key(models=[model], tpm_limit=100)
first: Final = _extract(gateway, model, uuid.uuid4().hex, key=key)
assert first.status_code == 200, first.text
assert len(wire.drain()) == 1
limited, upstream_calls = eventually(
lambda: (_extract(gateway, model, uuid.uuid4().hex, key=key), len(wire.drain())),
lambda answer: answer[0].status_code != 200,
seconds=70,
)
assert limited.status_code == 429, limited.text
assert upstream_calls == 0
def test_cloudflare_errors_keep_their_status_and_log_an_unbilled_failure(gateway: Gateway) -> None:
marker: Final = uuid.uuid4().hex
refusal: Final = json.dumps(
{
"result": None,
"success": False,
"errors": [{"code": 5006, "message": f"min above max {marker}"}],
"messages": [],
}
).encode()
with wire_server(lambda _: Reply(status=422, body=refusal)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
response: Final = _extract(gateway, model, marker)
assert response.status_code == 422, response.text
assert f"min above max {marker}" in response.text, response.text
assert len(wire.drain()) >= 1
row: Final = _spend_row(response.headers["x-litellm-call-id"])
assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == (
"failure",
"aextraction",
model,
0.0,
), row
def _owned_reply(request: Request) -> Reply:
body: Final = JSON_OBJECT.validate_json(request.body)
state: Final = body["state"]
assert isinstance(state, str), request.body
scenario: Final = _SCENARIO.search(state)
assert scenario is not None, state
if scenario.group(1) != "mask-out":
return _clairic_reply(request)
return _envelope(
{
"model": "clairic",
"answers": {
"vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)},
"contact_email": {"type": "text", "value": _EMAIL, "confidence": 0.9, "span": _email_span(state)},
},
"truncated": False,
"state_text": state,
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
}
)
def _owned_config(upstream_url: str) -> dict[str, JsonValue]:
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
email_filter: Final[list[JsonValue]] = [{"pattern_type": "prebuilt", "pattern_name": "email", "action": "MASK"}]
return {
**base,
"guardrails": [
{
"guardrail_name": _REQUEST_GUARDRAIL,
"litellm_params": {"guardrail": "litellm_content_filter", "mode": "pre_call", "patterns": email_filter},
},
{
"guardrail_name": _RESPONSE_GUARDRAIL,
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "post_call",
"patterns": email_filter,
},
},
],
"model_list": [
{
"model_name": _OWNED_MODEL,
"litellm_params": {"model": _CLAIRIC, "api_base": upstream_url, "api_key": _API_KEY},
}
],
}
@dataclass(frozen=True, slots=True)
class Rig:
proxy: OwnedProxy
upstream: Wire
@pytest.fixture(scope="module")
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
with wire_server(_owned_reply) as upstream:
directory: Final = tmp_path_factory.mktemp("clairic-extraction-owned")
config_path: Final = directory / "clairic-extraction.yaml"
config_path.write_text(yaml.safe_dump(_owned_config(upstream.url)))
with owned_proxy_process(
session_gateway,
directory,
{},
config=config_path,
workers=2,
extra_arguments=("--max_tokens", "256", "--temperature", "0.2"),
healthcheck=WorkerHealthcheck.LENIENT,
) as proxy:
yield Rig(proxy, upstream)
def _owned_extract(rig: Rig, state: str, **extra: JsonValue) -> httpx.Response:
return rig.proxy.gateway.request(
"POST", "/v1/extraction", {"model": _OWNED_MODEL, "state": state, "fields": _FIELDS, **extra}
)
def _applied_guardrails(response: httpx.Response) -> frozenset[str]:
return frozenset(name.strip() for name in response.headers.get("x-litellm-applied-guardrails", "").split(","))
@pytest.mark.timeout(_OWNED_TIMEOUT)
def test_cli_generation_defaults_never_reach_an_extraction_call(rig: Rig) -> None:
state: Final = _state(uuid.uuid4().hex)
response: Final = _owned_extract(rig, state)
assert response.status_code == 200, response.text
assert response.json() == _expected_response(_OWNED_MODEL, state)
(request,) = rig.upstream.drain()
assert _upstream_body(request) == {"model": "clairic", "state": state, "fields": _FIELDS}
@pytest.mark.timeout(_OWNED_TIMEOUT)
def test_a_request_guardrail_masks_the_state_before_clairic_and_drops_spans_into_it(rig: Rig) -> None:
state: Final = f"{_state(uuid.uuid4().hex, 'mask-in')} Questions go to {_EMAIL}."
response: Final = _owned_extract(rig, state, guardrails=[_REQUEST_GUARDRAIL])
assert response.status_code == 200, response.text
assert _REQUEST_GUARDRAIL in _applied_guardrails(response), response.headers
(request,) = rig.upstream.drain()
sent: Final = _upstream_body(request)
sent_state: Final = sent["state"]
assert isinstance(sent_state, str) and _EMAIL not in sent_state, sent
assert sent_state.startswith(state.removesuffix(f"{_EMAIL}.")) and "REDACTED" in sent_state, sent
assert sent == {"model": "clairic", "state": sent_state, "fields": _FIELDS}
answers: Final = object_value(response.json()["answers"])
assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers
@pytest.mark.timeout(_OWNED_TIMEOUT)
def test_a_response_guardrail_masks_text_answers_and_state_text(rig: Rig) -> None:
state: Final = f"{_state(uuid.uuid4().hex, 'mask-out')} Questions go to {_EMAIL}."
fields: Final[dict[str, JsonValue]] = {**_FIELDS, "contact_email": {"type": "text"}}
response: Final = rig.proxy.gateway.request(
"POST",
"/v1/extraction",
{
"model": _OWNED_MODEL,
"state": state,
"fields": fields,
"guardrails": [_RESPONSE_GUARDRAIL],
"options": {"return_state_text": True},
},
)
assert response.status_code == 200, response.text
assert _RESPONSE_GUARDRAIL in _applied_guardrails(response), response.headers
(request,) = rig.upstream.drain()
assert _upstream_body(request)["state"] == state
body: Final = response.json()
state_text: Final = body["state_text"]
assert isinstance(state_text, str) and _EMAIL not in state_text and "REDACTED" in state_text, body
answers: Final = object_value(body["answers"])
contact: Final = object_value(answers["contact_email"])
assert "span" not in contact, answers
masked: Final = contact["value"]
assert isinstance(masked, str) and _EMAIL not in masked and "REDACTED" in masked, answers
assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers

View file

@ -1,16 +1,113 @@
import json
import os
import uuid
from itertools import product
from pathlib import Path
from typing import Final
import httpx
import pytest
import yaml
from integration._support.client import Gateway, eventually, object_value
from integration._support.process import graceful_stop_seconds, owned_proxy
from integration._support.redis_process import owned_redis
from pydantic import JsonValue, TypeAdapter
from redis import Redis
import litellm
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.caching.redis_cache import RedisCache
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY, LITELLM_PROXY_BUDGET_NAME
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.proxy_server import update_cache
BUDGET_MODEL: Final = "proxy-budget-model"
CONFIG: Final = TypeAdapter(dict[str, JsonValue])
SECTION: Final = TypeAdapter(dict[str, JsonValue])
OWNED_PROXY_SECONDS: Final = 2 * graceful_stop_seconds() + 120
def _proxy_budget_config(gateway: Gateway, directory: Path, max_budget: float) -> Path:
loaded: Final = CONFIG.validate_python(
yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text())
)
model: Final[JsonValue] = {
"model_name": BUDGET_MODEL,
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_key": "integration-provider-key",
"api_base": f"{gateway.upstream_url}/v1",
"input_cost_per_token": 0.001,
"output_cost_per_token": 0.002,
},
}
path: Final = directory / "proxy_budget_config.yaml"
path.write_text(
yaml.safe_dump(
{
**loaded,
"model_list": [model],
"general_settings": {
**SECTION.validate_python(loaded["general_settings"]),
"user_api_key_cache_ttl": 600,
},
"litellm_settings": {
**SECTION.validate_python(loaded["litellm_settings"]),
"max_budget": max_budget,
"budget_duration": "30d",
},
}
)
)
return path
def _chat(proxy: Gateway, key: str, marker: str) -> httpx.Response:
return proxy.request(
"POST",
"/v1/chat/completions",
{"model": BUDGET_MODEL, "max_tokens": 20, "messages": [{"role": "user", "content": marker}]},
key=key,
)
def _served_id(proxy: Gateway, key: str) -> str:
response: Final = _chat(proxy, key, f"proxy budget {uuid.uuid4().hex}")
assert response.status_code == 200, response.text
identity: Final = response.json()["id"]
assert isinstance(identity, str), response.text
return identity
def _logged_spend(proxy: Gateway, request_ids: tuple[str, ...]) -> tuple[float, ...]:
rows: Final = eventually(
lambda: proxy.rows(
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s::text[])',
("{" + ",".join(request_ids) + "}",),
),
lambda found: len(found) == len(request_ids),
seconds=70,
)
return tuple(float(str(row["spend"])) for row in rows)
def _proxy_budget_spend(proxy: Gateway) -> float | None:
rows: Final = proxy.rows('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id = %s', (LITELLM_PROXY_BUDGET_NAME,))
return float(str(rows[0]["spend"])) if rows else None
def _counter_key(redis: Redis) -> str:
found: Final = tuple(str(key, "utf-8") for key in redis.scan_iter(match=f"*{GLOBAL_PROXY_SPEND_CACHE_KEY}"))
assert len(found) == 1, found
return found[0]
def _upstream_saw(proxy: Gateway, marker: str) -> bool:
with httpx.Client(base_url=proxy.upstream_url, timeout=5, trust_env=False) as upstream:
response: Final = upstream.get("/__observations")
response.raise_for_status()
return marker in json.dumps(object_value(response.json())["requests"])
async def _add_request_cost(cache: UserApiKeyCache, response_cost: float) -> None:
await update_cache(
@ -56,3 +153,63 @@ async def test_global_proxy_spend_adds_every_instance_cost_and_stays_cleared_aft
assert await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
finally:
await redis_cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
@pytest.mark.timeout(OWNED_PROXY_SECONDS)
def test_a_key_with_no_user_spends_toward_the_proxy_max_budget(gateway: Gateway, tmp_path: Path) -> None:
with (
owned_proxy(gateway, tmp_path, {}, config=_proxy_budget_config(gateway, tmp_path, max_budget=0.01)) as owned,
owned.scenario() as scenario,
):
key: Final = scenario.key(models=[BUDGET_MODEL])
(spent,) = _logged_spend(owned, (_served_id(owned, key),))
assert spent > 0.01, spent
eventually(
lambda: _proxy_budget_spend(owned),
lambda recorded: recorded is not None and recorded > 0.01,
seconds=70,
)
marker: Final = f"over the proxy budget {uuid.uuid4().hex}"
refused: Final = _chat(owned, key, marker)
assert refused.status_code != 200, refused.text
assert object_value(refused.json()["error"])["type"] == "budget_exceeded", refused.text
assert not _upstream_saw(owned, marker), refused.text
@pytest.mark.timeout(2 * OWNED_PROXY_SECONDS)
def test_two_proxies_add_every_request_to_one_proxy_spend_counter(gateway: Gateway, tmp_path: Path) -> None:
config: Final = _proxy_budget_config(gateway, tmp_path, max_budget=1000.0)
with owned_redis(tmp_path / "redis") as cache, Redis(host=cache.host, port=cache.port) as redis:
shared_redis: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)}
with owned_proxy(gateway, tmp_path / "first", shared_redis, config=config) as first:
assert first.database_url is not None
with (
owned_proxy(
gateway, tmp_path / "second", {**shared_redis, "DATABASE_URL": first.database_url}, config=config
) as second,
first.scenario() as scenario,
):
user_key: Final = scenario.key(user_id=scenario.user(), models=[BUDGET_MODEL])
userless_key: Final = scenario.key(models=[BUDGET_MODEL])
counter: Final = _counter_key(redis)
ttl_before: Final = redis.ttl(counter)
request_ids: Final = tuple(
_served_id(proxy, key) for key, proxy in product((user_key, userless_key), (first, second))
)
total: Final = sum(_logged_spend(first, request_ids))
persisted: Final = eventually(
lambda: _proxy_budget_spend(first),
lambda spend: spend is not None and spend == pytest.approx(total),
seconds=70,
)
assert persisted == pytest.approx(total)
eventually(
lambda: redis.get(counter),
lambda shared: shared is not None and float(str(shared, "utf-8")) == pytest.approx(total),
seconds=70,
)
assert 0 < redis.ttl(counter) <= ttl_before

View file

@ -105,6 +105,24 @@ def test_openai_image_edit_sync_accepts_bytesio_images(respx_mock: respx.MockRou
assert _multipart_image_parts(route.calls[0].request) == (_FIRST_IMAGE,)
def test_openai_image_edit_without_a_model_leaves_model_out_of_the_request(
respx_mock: respx.MockRouter, httpx_transport: None
) -> None:
route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock(
return_value=httpx.Response(200, json=_EDIT_RESPONSE)
)
litellm.image_edit(
prompt="combine the reference images",
image=io.BytesIO(_FIRST_IMAGE),
custom_llm_provider="openai",
api_key="fake-key",
)
assert route.call_count == 1
assert b'name="model"' not in route.calls[0].request.read()
@pytest.mark.asyncio
async def test_openai_image_edit_accepts_bytesio_images(respx_mock: respx.MockRouter, httpx_transport: None) -> None:
route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock(

View file

@ -300,7 +300,7 @@ async def test_aimage_generation_ignores_a_non_string_base_url_like_the_sync_pat
)
response: Final = await litellm.aimage_generation(
model="dall-e-3",
model="gpt-image-2.5-sunburst",
prompt="a red circle",
api_base=api_base,
base_url=httpx.URL(api_base),
@ -688,3 +688,24 @@ async def test_chatgpt_images_forward_client_login_and_workspace(
assert "cookie" not in request.headers
assert "x-litellm-api-key" not in request.headers
assert json.loads(request.content) == {"model": "gpt-image-2", "prompt": "A blue circle"}
def test_image_generation_without_a_model_or_provider_fails_before_calling_any_provider(
respx_mock: respx.MockRouter,
) -> None:
with pytest.raises(litellm.BadRequestError, match="model is required"):
litellm.image_generation(prompt="a red circle", api_key="fake_openai_api_key")
assert not respx_mock.calls
def test_topaz_image_variation_works_without_a_model(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post("https://api.topazlabs.com/image/v1/enhance").mock(
return_value=httpx.Response(200, content=b"enhanced-image", headers={"content-type": "image/png"})
)
response: Final = litellm.image_variation(image=b"source-image", custom_llm_provider="topaz", api_key="fake-key")
assert route.call_count == 1
assert response.data is not None
assert response.data[0].b64_json == base64.b64encode(b"enhanced-image").decode()

View file

@ -7,7 +7,7 @@ from typing import Dict, Final, List
import pytest
import litellm
from litellm.router_utils.batch_utils import (
from litellm.litellm_core_utils.batch_utils import (
InMemoryFile,
get_router_metadata_variable_name,
parse_jsonl_with_embedded_newlines,
@ -116,7 +116,7 @@ def test_router_metadata_variable_name():
def test_non_json_input():
"""Test that replace_model_in_jsonl returns original content for non-JSON input"""
from litellm.router_utils.batch_utils import replace_model_in_jsonl
from litellm.litellm_core_utils.batch_utils import replace_model_in_jsonl
# Test with non-JSON string
non_json_str = "This is not a JSON string"
@ -138,7 +138,7 @@ def test_non_json_input():
def test_should_replace_model_in_jsonl():
"""Test that should_replace_model_in_jsonl returns the correct value"""
from litellm.router_utils.batch_utils import should_replace_model_in_jsonl
from litellm.litellm_core_utils.batch_utils import should_replace_model_in_jsonl
assert should_replace_model_in_jsonl(purpose="batch") is True
assert should_replace_model_in_jsonl(purpose="batch", passthrough=True) is False
@ -340,7 +340,7 @@ def test_replace_model_in_jsonl_with_embedded_newlines():
def test_is_batch_retrieve_call_type_matches_only_batch_retrieves():
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
from litellm.types.utils import CallTypes
assert is_batch_retrieve_call_type(CallTypes.aretrieve_batch.value) is True
@ -389,7 +389,8 @@ def setup_and_teardown():
)
def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str, cost: float, eligible: bool) -> None:
from openai.types import Batch
from litellm.router_utils.batch_utils import batch_id_to_charge_once
from litellm.litellm_core_utils.batch_utils import batch_id_to_charge_once
batch: Final = Batch(
id="batch-budget", completion_window="24h", created_at=1, endpoint="/v1/chat/completions",
@ -401,7 +402,7 @@ def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str,
def test_batch_charge_markers_keep_counters_and_batches_independent() -> None:
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.router_utils.batch_utils import batch_charged_once_marker_key
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key
cache: Final = InMemoryCache()
cache.set_cache(batch_charged_once_marker_key("provider-spend", "first-batch"), 1)

View file

@ -20,27 +20,9 @@ from litellm.llms.azure.image_generation.http_utils import (
azure_deployment_image_generation_json_body,
)
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.azure.image_generation import (
AzureDallE3ImageGenerationConfig,
get_azure_image_generation_config,
)
from litellm.utils import get_optional_params_image_gen
@pytest.mark.parametrize(
"received_model, expected_config",
[
("dall-e-3", AzureDallE3ImageGenerationConfig),
("dalle-3", AzureDallE3ImageGenerationConfig),
("openai_dall_e_3", AzureDallE3ImageGenerationConfig),
],
)
def test_azure_image_generation_config(received_model, expected_config):
assert isinstance(
get_azure_image_generation_config(received_model), expected_config
)
def test_azure_deployment_image_generation_json_body():
"""Deployment-scoped Azure image URL must not send ``model`` in JSON."""
api = (

View file

@ -265,7 +265,7 @@ class TestImageGenerationEntryPointHeaders:
test_headers = {"cf-aig-authorization": "Bearer my-secret"}
await litellm.aimage_generation(
model="dall-e-3",
model="gpt-image-2.5-sunburst",
prompt="A white cat",
extra_headers=test_headers,
client=mock_openai_client,

View file

@ -46,12 +46,12 @@ API_FUNCTION_PARAMS = [
(
"image_generation",
False,
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
{"model": "gpt-image-2.5-sunburst", "prompt": "A beautiful sunset over mountains"},
),
(
"image_generation",
True,
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
{"model": "gpt-image-2.5-sunburst", "prompt": "A beautiful sunset over mountains"},
),
(
"speech",
@ -84,11 +84,7 @@ async def test_openai_client_reuse(function_name, is_async, args):
"""
# Determine which client class to mock based on whether the test is async
client_path = (
"litellm.llms.openai.openai.AsyncOpenAI"
if is_async
else "litellm.llms.openai.openai.OpenAI"
)
client_path = "litellm.llms.openai.openai.AsyncOpenAI" if is_async else "litellm.llms.openai.openai.OpenAI"
# Create the appropriate patches
with (
@ -98,9 +94,7 @@ async def test_openai_client_reuse(function_name, is_async, args):
):
# Setup the mock to return None first time (cache miss) then a client for subsequent calls
mock_client = MagicMock()
mock_get_cache.side_effect = [None] + [
mock_client
] * 9 # First call returns None, rest return the mock client
mock_get_cache.side_effect = [None] + [mock_client] * 9 # First call returns None, rest return the mock client
# Make 10 API calls
for _ in range(10):
@ -118,9 +112,9 @@ async def test_openai_client_reuse(function_name, is_async, args):
pass
# Verify client was created only once
assert (
mock_client_class.call_count == 1
), f"{'Async' if is_async else ''}OpenAI client should be created only once"
assert mock_client_class.call_count == 1, (
f"{'Async' if is_async else ''}OpenAI client should be created only once"
)
# Verify the client was cached
assert mock_set_cache.call_count == 1, "Client should be cached once"
@ -144,12 +138,8 @@ def test_precomputed_init_params_match_inspect_signature():
_OPENAI_INIT_PARAMS,
)
expected_openai = tuple(
p for p in inspect.signature(OpenAI.__init__).parameters if p != "self"
)
expected_azure = tuple(
p for p in inspect.signature(AzureOpenAI.__init__).parameters if p != "self"
)
expected_openai = tuple(p for p in inspect.signature(OpenAI.__init__).parameters if p != "self")
expected_azure = tuple(p for p in inspect.signature(AzureOpenAI.__init__).parameters if p != "self")
assert _OPENAI_INIT_PARAMS == expected_openai
assert _AZURE_OPENAI_INIT_PARAMS == expected_azure
@ -500,7 +490,10 @@ async def test_sdk_api_factories_keep_httpx_transport_and_request_timeouts(api:
moved: Final = mock.get(host="sdk-default.example", path="/v1/moved-models").mock(
return_value=httpx.Response(
200,
json={"object": "list", "data": [{"id": "sdk-compat", "object": "model", "created": 0, "owned_by": "t"}]},
json={
"object": "list",
"data": [{"id": "sdk-compat", "object": "model", "created": 0, "owned_by": "t"}],
},
)
)
sdk_client: Final = _sdk_api_client(api, is_async, timeout)
@ -542,7 +535,10 @@ async def test_azure_gateway_and_image_variations_use_the_callers_async_session(
assert azure_client is not None
assert azure_client._client is session
response: Final = await aimage_variation(
image=BytesIO(b"image-bytes"), api_key="transport-only", api_base="https://sdk-default.example/v1"
image=BytesIO(b"image-bytes"),
custom_llm_provider="openai",
api_key="transport-only",
api_base="https://sdk-default.example/v1",
)
assert response.data[0].url == "https://example.com/image.png"
assert not session.is_closed

View file

@ -6,12 +6,14 @@ from datetime import datetime, timedelta
from queue import SimpleQueue
from threading import get_ident
from typing import Final
from unittest.mock import MagicMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import requests
import responses
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import (
VertexAIPartnerModelsAnthropicMessagesConfig,
)
@ -94,6 +96,85 @@ async def test_messages_auth_load_and_refresh_leave_the_event_loop(stream: bool,
assert headers == {"Authorization": "Bearer incoming", "anthropic-beta": "retained-beta", "x-request-id": "req-1"}
def _vertex_client() -> MagicMock:
vertex_response: Final = MagicMock()
vertex_response.json.return_value = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"content": [{"type": "text", "text": "hi"}],
"model": "claude-sonnet-5",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
}
client: Final = MagicMock(spec=AsyncHTTPHandler)
client.post = AsyncMock(return_value=vertex_response)
return client
async def _send_to_vertex(client: MagicMock, caller_headers: Mapping[str, str], drop_params: bool) -> None:
with responses.RequestsMock(assert_all_requests_are_fired=False) as oauth:
oauth.add(responses.POST, "https://oauth2.googleapis.com/token", json={"access_token": "vertex-token", "expires_in": 3600})
await litellm.anthropic.messages.acreate(
model="vertex_ai/claude-sonnet-5",
messages=[{"role": "user", "content": "hi"}],
max_tokens=16,
vertex_ai_project="version-project",
vertex_ai_location="global",
vertex_credentials=json.dumps(
{
"type": "authorized_user",
"client_id": "version-header-client",
"client_secret": "synthetic-secret",
"refresh_token": "synthetic-refresh",
}
),
client=client,
drop_params=drop_params,
provider_specific_header={
"custom_llm_provider": "vertex_ai",
"extra_headers": {**caller_headers, "x-request-id": "req-1"},
},
)
@pytest.mark.parametrize(
("caller_headers", "drop_params", "body_version"),
(
({"anthropic-version": "2026-10-21"}, False, "2026-10-21"),
({"Anthropic-Version": "2026-10-21"}, False, "2026-10-21"),
({"anthropic-version": "vertex-2023-10-16"}, False, "vertex-2023-10-16"),
({"anthropic-version": "2023-06-01"}, False, "vertex-2023-10-16"),
({"anthropic-version": "2023-01-01"}, False, "vertex-2023-10-16"),
({"anthropic-version": " "}, False, "vertex-2023-10-16"),
({}, False, "vertex-2023-10-16"),
({"anthropic-version": "latest"}, True, "vertex-2023-10-16"),
),
)
@pytest.mark.asyncio
async def test_caller_anthropic_version_reaches_vertex_only_in_the_body(
caller_headers: dict[str, str], drop_params: bool, body_version: str
) -> None:
client: Final = _vertex_client()
await _send_to_vertex(client, caller_headers, drop_params)
sent: Final = client.post.call_args.kwargs
sent_header_names: Final = {name.lower() for name in sent["headers"]}
assert "anthropic-version" not in sent_header_names
assert sent["headers"]["x-request-id"] == "req-1"
assert json.loads(sent["data"])["anthropic_version"] == body_version
@pytest.mark.parametrize("requested", ("2025-01-01", "latest"))
@pytest.mark.asyncio
async def test_unsupported_anthropic_version_is_refused_without_drop_params(requested: str) -> None:
client: Final = _vertex_client()
with pytest.raises(litellm.BadRequestError, match=f"anthropic-version={requested}. To drop unsupported params"):
await _send_to_vertex(client, {"anthropic-version": requested}, drop_params=False)
client.post.assert_not_awaited()
def test_validate_environment_uses_vertex_ai_location():
config = VertexAIPartnerModelsAnthropicMessagesConfig()
headers = {}

View file

@ -170,7 +170,7 @@ async def test_preview_and_serving_share_default_model_resolution(
explicit_default: str | None,
expected: str,
):
from litellm.router_utils.auto_router_model_naming import validate_complexity_router_config_write
from litellm.router_strategy.auto_router.auto_router_model_naming import validate_complexity_router_config_write
from litellm.types.management_endpoints.auto_router_endpoints import ComplexityRouterConfigValidationRequest
config: Final = {
@ -1240,7 +1240,7 @@ class TestAutoRouterBenchmarks:
def test_the_listed_kinds_match_the_router_types_traffic_can_record(self):
from typing import get_args, get_type_hints
from litellm.router_utils.auto_router_model_naming import StrategyRouterKind
from litellm.router_strategy.auto_router.auto_router_model_naming import StrategyRouterKind
from litellm.types.utils import StandardLoggingRoutingDecision
readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"]

View file

@ -1,6 +1,6 @@
import inspect
import asyncio
import contextlib
import inspect
import json
import socket
from collections.abc import Iterator, Mapping
@ -1461,10 +1461,10 @@ class TestDeleteModelClearsRouterRegistry:
add_deployment never restores, so an unguarded pop would make it permanently
unroutable (the same cross-tenant DoS clear_cache was hardened against).
"""
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete
model_id = "regular-del-1"
admin_user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
@ -2739,11 +2739,11 @@ class TestAddAndDeleteModelLifecycle:
- Delete same model again → raises (model not found)
"""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
add_new_model,
delete_model as delete_model_endpoint,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
delete_model as delete_model_endpoint,
)
model_id = "lifecycle-test-model-123"
@ -2833,6 +2833,8 @@ class TestDeleteTeamBYOKModelGhost:
async def test_delete_strips_public_name_and_refreshes_cache(self):
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -2916,6 +2918,8 @@ class TestDeleteTeamBYOKModelGhost:
alias cleanup (delete_team_model_alias), preserving legacy behavior."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -2984,6 +2988,8 @@ class TestDeleteTeamBYOKModelGhost:
team.models when one replica is deleted but a sibling still backs it."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -3065,6 +3071,8 @@ class TestDeleteTeamBYOKModelGhost:
deployment still serves it."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -3145,6 +3153,8 @@ class TestDeleteTeamBYOKModelGhost:
the alias would break routing that works."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -3252,6 +3262,8 @@ class TestDeleteModelTeamAuth:
async def test_proxy_admin_can_delete_model_when_team_deleted(self):
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
@ -3287,6 +3299,8 @@ class TestDeleteModelTeamAuth:
"""A missing team must never let a non-admin delete the orphan (no fail-open)."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
from litellm.proxy.proxy_server import ProxyException
@ -3322,6 +3336,8 @@ class TestDeleteModelTeamAuth:
"""The auth check must not add a redundant team query on the live-team path."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)
from litellm.proxy.proxy_server import ProxyException
@ -4392,7 +4408,6 @@ class TestModelInfoCostMapEchoFilter:
def test_echoed_cost_map_metadata_is_not_persisted(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4418,7 +4433,6 @@ class TestModelInfoCostMapEchoFilter:
def test_an_edited_value_survives_the_echo_filter(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4483,7 +4497,6 @@ class TestModelInfoCostMapEchoFilter:
def test_metadata_without_a_cost_map_key_is_persisted(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
from litellm.types.utils import echoed_cost_map_fields
@ -4513,7 +4526,6 @@ class TestModelInfoCostMapEchoFilter:
def test_a_stored_mode_survives_an_echoed_save(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4536,7 +4548,6 @@ class TestModelInfoCostMapEchoFilter:
def test_resetting_an_override_to_the_cost_map_value_removes_it(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4560,7 +4571,6 @@ class TestModelInfoCostMapEchoFilter:
def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4586,7 +4596,6 @@ class TestModelInfoCostMapEchoFilter:
from types import MappingProxyType
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4611,7 +4620,6 @@ class TestModelInfoCostMapEchoFilter:
def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4653,7 +4661,6 @@ class TestModelInfoCostMapEchoFilter:
def test_base_model_wins_over_litellm_params_model_for_the_lookup(self):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -4681,7 +4688,6 @@ class TestModelInfoCostMapEchoFilter:
def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch):
import litellm
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
@ -6431,7 +6437,7 @@ class TestStrategyRouterWriteValidation:
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,
_auto_router_capability_slot,
)
from litellm.router_utils.auto_router_model_naming import gated_capabilities_of
from litellm.router_strategy.auto_router.auto_router_model_naming import gated_capabilities_of
capabilities: Final = gated_capabilities_of(effective_params)
@ -6568,7 +6574,7 @@ class TestStrategyRouterWriteValidation:
from fastapi import HTTPException
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
configs = {
"_TUNED_A": self._TUNED_A,
@ -6624,7 +6630,7 @@ class TestStrategyRouterWriteValidation:
from litellm.proxy import proxy_server
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
fake = self._FakeDb([])
fake.tx_obj.litellm_proxymodeltable.find_many = AsyncMock(
@ -6669,7 +6675,7 @@ class TestStrategyRouterWriteValidation:
row id before the slot is entered, so a second tuned router is refused before its DB write."""
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
baselines = snapshot_tuning_baselines([self._db_router_row("a", self._TUNED_A)])
@ -8980,7 +8986,10 @@ class TestTeamMemberAutoRouterWrites:
from fastapi import HTTPException
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
from litellm.proxy.management_helpers.auto_router_permissions import MemberAutoRouterWrite, validate_member_auto_router_config
from litellm.proxy.management_helpers.auto_router_permissions import (
MemberAutoRouterWrite,
validate_member_auto_router_config,
)
row: Final = self._row()
database: Final = self._database(self._team(), row)
@ -10677,6 +10686,8 @@ class TestFederationGateScopesToWhatTheWriteTouches:
and the team that owns the model must be able to take it off their page."""
from litellm.proxy.management_endpoints.model_management_endpoints import (
ModelInfoDelete,
)
from litellm.proxy.management_endpoints.model_management_endpoints import (
delete_model as delete_model_endpoint,
)

View file

@ -1,16 +1,15 @@
from collections.abc import Mapping
from typing import Final
from types import SimpleNamespace
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from typing import Final
import pytest
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
from litellm.proxy.management_helpers.auto_router_availability import (
auto_router_availability,
build_auto_router_catalog,
)
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
def deployment(

View file

@ -33,9 +33,9 @@ from pydantic import BaseModel
from typing_extensions import TypedDict
import litellm.proxy.proxy_server as ps
from litellm.proxy.management_helpers.audit_logs import track_audit_task
from litellm.proxy.proxy_server import (
ProxyStartupEvent,
initialize_shared_aiohttp_session,
_resolve_pydantic_type,
_resolve_typed_dict_type,
cleanup_router_config_variables,
@ -43,6 +43,7 @@ from litellm.proxy.proxy_server import (
get_litellm_model_info,
initialize,
initialize_from_worker_config,
initialize_shared_aiohttp_session,
load_from_azure_key_vault,
proxy_shutdown_event,
proxy_startup_event,
@ -50,8 +51,6 @@ from litellm.proxy.proxy_server import (
save_worker_config,
)
from litellm.proxy.management_helpers.audit_logs import track_audit_task
from .conftest import normalize
# ---------------------------------------------------------------------------
@ -1165,7 +1164,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database():
@pytest.mark.asyncio
async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row():
from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT
prisma_client = MagicMock()
prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None)
@ -1186,7 +1185,10 @@ async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row():
@pytest.mark.asyncio
async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart():
from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
mutable_tuned_identities,
snapshot_tuning_baselines,
)
deployments = [
{

View file

@ -3,9 +3,7 @@ from typing import Final
import pytest
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS
from litellm.router_utils.auto_router_model_naming import (
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
capability_limit_violation,
carries_complexity_router_settings,
@ -18,6 +16,8 @@ from litellm.router_utils.auto_router_model_naming import (
validate_complexity_router_config_write,
validate_strategy_router_model_write,
)
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS
COMPLEXITY_FIELDS = frozenset({"complexity_router_config"})
SEMANTIC_FIELDS = frozenset({"auto_router_config", "auto_router_default_model", "auto_router_embedding_model"})
@ -596,7 +596,7 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec
],
)
def test_is_complexity_router_model(model: str | None, expected: bool) -> None:
from litellm.router_utils.auto_router_model_naming import is_complexity_router_model
from litellm.router_strategy.auto_router.auto_router_model_naming import is_complexity_router_model
assert is_complexity_router_model(model) is expected

View file

@ -7,8 +7,7 @@ from typing import Final
import pytest
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
from litellm.router_utils.auto_router_tuning_baseline import (
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
DEFAULT_TUNING_FINGERPRINT,
HEURISTIC_V1_TUNING_FIELDS,
heuristic_v1_router_fingerprint,
@ -19,6 +18,7 @@ from litellm.router_utils.auto_router_tuning_baseline import (
tuning_limit_violation,
tuning_quota_violation,
)
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
_TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}
_ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"}

View file

@ -1,6 +1,6 @@
from typing import Final
from litellm.router_utils.baseline_request import baseline_request, capture_baseline_parameters
from litellm.router_strategy.auto_router.baseline_request import baseline_request, capture_baseline_parameters
def test_baseline_snapshot_owns_nested_caller_settings_and_overrides_routed_settings() -> None:

View file

@ -24,14 +24,6 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm import Router
from litellm.integrations.custom_logger import CustomLogger
from litellm.types.utils import StandardLoggingRoutingDecision
from litellm.router_utils.auto_router_model_naming import (
CUSTOMIZATION_CAPABILITY,
GATED_AUTO_ROUTER_CAPABILITIES,
HEURISTIC_V2_CAPABILITY,
count_capability_routers,
)
from litellm._logging import verbose_router_logger
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
@ -42,7 +34,19 @@ from litellm.constants import (
RETURN_RAW_MODEL_NAME_METADATA_KEY,
SESSION_ID_GENERATED_METADATA_KEY,
)
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.router import as_output_cap, stamped_client_ceiling
from litellm.router_strategy.auto_router.auto_router_model_naming import (
CUSTOMIZATION_CAPABILITY,
GATED_AUTO_ROUTER_CAPABILITIES,
HEURISTIC_V2_CAPABILITY,
count_capability_routers,
)
from litellm.router_strategy.complexity_router.capability_classifier import (
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT,
CapabilityClassifierVerdict,
)
from litellm.router_strategy.complexity_router.complexity_router import (
_CLASSIFICATION_CURRENT_MESSAGE_ONLY,
_CLASSIFICATION_WITH_CONVERSATION,
@ -58,18 +62,14 @@ from litellm.router_strategy.complexity_router.complexity_router import (
classification_system_prompt,
custom_tier_classification_prompt,
)
from litellm.router_strategy.complexity_router.capability_classifier import (
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT,
CapabilityClassifierVerdict,
)
from litellm.router_strategy.complexity_router.config import (
CapabilityCalibrationConfig,
CapabilityClassifierConfig,
DEFAULT_CLASSIFICATION_RUBRIC,
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
DEFAULT_COMPLEXITY_CONFIG,
DEFAULT_TECHNICAL_KEYWORDS,
TIER_SEVERITY_ORDER,
CapabilityCalibrationConfig,
CapabilityClassifierConfig,
ClassificationRubric,
ClassifierLLMConfig,
ComplexityRouterConfig,
@ -83,10 +83,12 @@ from litellm.router_strategy.complexity_router.jev_classifier import (
JevUsage,
)
from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
from litellm.router_strategy.complexity_router.tier_predictor import (
TierGlobalStatistic,
TrainedTierArtifact,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.router import (
Deployment,
LiteLLM_Params,
@ -94,10 +96,7 @@ from litellm.types.router import (
RouterErrors,
TaggedPreRoutingStrategy,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.types.utils import StandardLoggingRoutingDecision
requires_semantic_router = pytest.mark.skipif(
sys.version_info >= (3, 14), reason="The semantic-router extra excludes Python 3.14"
@ -16299,7 +16298,6 @@ class TestTierHealthFailover:
tests can prove the probe passes a kwargs copy and forwards the prompt arguments.
"""
import litellm as litellm_module
from litellm.types.router import RouterRateLimitError
probed_kwargs = []

View file

@ -4,11 +4,12 @@ from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
import litellm
from pydantic import ValidationError
import litellm
from litellm import ModelResponse, Router
from litellm.caching.dual_cache import DualCache
from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
@ -21,7 +22,6 @@ from litellm.router_strategy.complexity_router.llm_v2 import (
LLMV2Verdict,
llm_v2_response_format,
)
from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies
from litellm.types.llms.openai import ResponsesAPIResponse

View file

@ -1969,7 +1969,7 @@ ALIAS_CASES: Final = (
("litellm.router_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True),
("litellm.router_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False),
(
"litellm.router_utils.batch_utils",
"litellm.litellm_core_utils.batch_utils",
"",
"_get_router_metadata_variable_name",
"get_router_metadata_variable_name",

View file

@ -4,20 +4,21 @@ import gc
import json
import os
import weakref
from datetime import datetime, timezone
from collections.abc import Iterator, Mapping
from datetime import datetime, timezone
from types import MethodType
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import litellm
import pytest
import litellm
from litellm import Router
from litellm.caching.dual_cache import DualCache
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
from litellm.types.litellm_params import RoutingStrategyName
@ -801,10 +802,11 @@ async def test_deployment_callback_on_failure(model_list):
def test_deployment_callback_respects_cooldown_time(model_list):
"""Ensure per-model cooldown_time is honored even when exception headers are present."""
import httpx
import time
from unittest.mock import patch
import httpx
router = Router(model_list=model_list)
class FakeException(Exception):
@ -1811,8 +1813,8 @@ def test_get_allowed_fails_from_policy(model_list, exception_type, exception_nam
def test_initialize_alerting(model_list):
"""Test if the 'initialize_alerting' function is working correctly"""
from litellm.types.router import AlertingConfig
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.types.router import AlertingConfig
router = Router(
model_list=model_list, alerting_config=AlertingConfig(webhook_url="test")
@ -2245,6 +2247,7 @@ def test_deployments_by_pattern(model_list):
)
def test_pattern_match_deployment_set_model_name(user_request_model, model_name, litellm_model, expected_model):
from re import Match
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
pattern_router = PatternMatchRouter()
@ -2702,7 +2705,7 @@ def test_handle_clientside_credential_metadata_loading(model_list, function_name
)
def test_handle_clientside_credential_metadata_variable_name(model_list, function_name, metadata_key):
"""Test that _handle_clientside_credential uses the correct metadata variable name based on function name"""
from litellm.router_utils.batch_utils import get_router_metadata_variable_name
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
router = Router(model_list=model_list)

View file

@ -0,0 +1,58 @@
"""
The router surface fence (tests/code_coverage_tests/router_surface_check.py) must see every way
outside code reaches a Router member, or new dependencies slip past the shrink-only allowlist.
"""
import ast
import importlib.util
import pathlib
import pytest
_CHECK = pathlib.Path(__file__).parents[2] / "code_coverage_tests" / "router_surface_check.py"
@pytest.fixture(scope="module")
def surface_check():
spec = importlib.util.spec_from_file_location("router_surface_check", _CHECK)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _uses(surface_check, src: str) -> set:
return surface_check.member_uses(ast.parse(src), surface_check.router_members())
def test_instance_and_dynamic_access_are_uses(surface_check):
assert _uses(surface_check, "llm_router.get_model_list()\ngetattr(router, 'model_list')") == {
"get_model_list",
"model_list",
}
def test_class_level_calls_are_uses(surface_check):
assert _uses(surface_check, "litellm.Router.get_valid_args()") == {"get_valid_args"}
assert _uses(surface_check, "from litellm import Router\nRouter.get_valid_args()") == {"get_valid_args"}
def test_container_file_endpoints_registered_by_name_are_members(surface_check):
members = surface_check.router_members()
assert {"aupload_container_file", "upload_container_file", "alist_container_files"} <= members
assert _uses(surface_check, "await llm_router.aupload_container_file(file=f)") == {"aupload_container_file"}
def test_unlisted_class_level_use_fails_the_fence(surface_check):
found = {
"members": {"outside": {"get_valid_args": {"litellm/new_module.py"}}, "router_family": {}},
"router_utils_imports": {},
}
allow = {"members_outside": [], "members_router_family": [], "router_utils_imports_outside": []}
errors = surface_check.compare(allow, found)
assert len(errors) == 1
assert "'get_valid_args' in litellm/new_module.py" in errors[0]
def test_unrelated_receivers_are_not_uses(surface_check):
assert _uses(surface_check, "client.get_model_list()\nRouterConfig.model_list") == set()