mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge remote-tracking branch 'origin/main' into litellm_claude_code_marketplace_auth_required
This commit is contained in:
commit
f155703e59
93 changed files with 2390 additions and 829 deletions
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
|
|
@ -2,3 +2,4 @@
|
|||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
/.github/workflows/zizmor.yml @yuneng-berri
|
||||
|
|
|
|||
5
.github/workflows/cost-map-guard.yml
vendored
5
.github/workflows/cost-map-guard.yml
vendored
|
|
@ -1,7 +1,7 @@
|
|||
name: Cost map guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
|
||||
pull_request_target:
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
|
|
@ -19,6 +19,7 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head and its merge base
|
||||
id: revisions
|
||||
|
|
|
|||
6
.github/workflows/spend-logs-guard.yml
vendored
6
.github/workflows/spend-logs-guard.yml
vendored
|
|
@ -1,7 +1,7 @@
|
|||
name: Spend logs guard
|
||||
|
||||
on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's diff is read as data and never executed
|
||||
pull_request_target:
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ jobs:
|
|||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.ref }}
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
persist-credentials: false
|
||||
- name: Fetch the pull request head, its merge base and its reviews
|
||||
id: revisions
|
||||
|
|
|
|||
3
.github/workflows/test-linting.yml
vendored
3
.github/workflows/test-linting.yml
vendored
|
|
@ -561,6 +561,9 @@ jobs:
|
|||
- name: router_code_coverage
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
|
||||
|
||||
- name: router_surface_check
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/router_surface_check.py
|
||||
|
||||
- name: test_chat_completion_imports
|
||||
run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
|
||||
|
||||
|
|
|
|||
17
.github/workflows/zizmor.yml
vendored
17
.github/workflows/zizmor.yml
vendored
|
|
@ -27,10 +27,15 @@ jobs:
|
|||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install zizmor
|
||||
run: |
|
||||
curl -fsSL --retry 3 -o "$RUNNER_TEMP/zizmor.tar.gz" \
|
||||
https://github.com/zizmorcore/zizmor/releases/download/v1.31.1/zizmor-x86_64-unknown-linux-gnu.tar.gz
|
||||
echo "d185f726bdaf92df78273217dfdd24ebcd6b0ef48cf3ce776ff5762b6e138c99 $RUNNER_TEMP/zizmor.tar.gz" | sha256sum -c -
|
||||
tar xzf "$RUNNER_TEMP/zizmor.tar.gz" -C "$RUNNER_TEMP" zizmor
|
||||
|
||||
- name: Run zizmor
|
||||
uses: zizmorcore/zizmor-action@5f14fd08f7cf1cb1609c1e344975f152c7ee938d # v0.5.6
|
||||
with:
|
||||
version: "1.24.1"
|
||||
min-severity: medium
|
||||
advanced-security: false
|
||||
annotations: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
"$RUNNER_TEMP/zizmor" --no-ignores --no-config --min-severity=medium --format=github .
|
||||
|
|
|
|||
5
litellm-rust/Cargo.lock
generated
5
litellm-rust/Cargo.lock
generated
|
|
@ -4387,12 +4387,17 @@ name = "litellm-router"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-config",
|
||||
"litellm-core-utils",
|
||||
"litellm-inference",
|
||||
"litellm-inference-messages",
|
||||
"litellm-router-types",
|
||||
"rstest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -1,13 +1,25 @@
|
|||
pub fn parse_str_bool(value: &str) -> Option<bool> {
|
||||
let token = value.trim_matches(|character: char| {
|
||||
fn python_strip(value: &str) -> &str {
|
||||
value.trim_matches(|character: char| {
|
||||
character.is_whitespace() || matches!(character, '\u{1c}'..='\u{1f}')
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
pub fn parse_str_bool(value: &str) -> Option<bool> {
|
||||
let token = python_strip(value);
|
||||
if token.eq_ignore_ascii_case("true") {
|
||||
return Some(true);
|
||||
}
|
||||
token.eq_ignore_ascii_case("false").then_some(false)
|
||||
}
|
||||
|
||||
pub fn parse_pydantic_bool(value: &str) -> Option<bool> {
|
||||
match python_strip(value).to_ascii_lowercase().as_str() {
|
||||
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
|
||||
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `redis-py` string Booleans: only `1`, `true`, and `yes` (case-insensitive) are true.
|
||||
pub fn parse_redis_bool(value: &str) -> bool {
|
||||
value == "1" || value.eq_ignore_ascii_case("true") || value.eq_ignore_ascii_case("yes")
|
||||
|
|
@ -34,4 +46,27 @@ mod tests {
|
|||
) {
|
||||
assert_eq!(parse_str_bool(input), expected, "{input:?}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::yes(" YES ", Some(true))]
|
||||
#[case::on("On", Some(true))]
|
||||
#[case::one("1", Some(true))]
|
||||
#[case::t("t", Some(true))]
|
||||
#[case::y("y", Some(true))]
|
||||
#[case::control_whitespace_true("\u{1c}TRUE\u{1f}", Some(true))]
|
||||
#[case::no("No", Some(false))]
|
||||
#[case::off("OFF", Some(false))]
|
||||
#[case::zero("0", Some(false))]
|
||||
#[case::f("F", Some(false))]
|
||||
#[case::n("n", Some(false))]
|
||||
#[case::false_word("false", Some(false))]
|
||||
#[case::two("2", None)]
|
||||
#[case::empty("", None)]
|
||||
#[case::unknown("maybe", None)]
|
||||
fn boolean_words_follow_pydantic_lax_bool_parsing(
|
||||
#[case] input: &str,
|
||||
#[case] expected: Option<bool>,
|
||||
) {
|
||||
assert_eq!(parse_pydantic_bool(input), expected, "{input:?}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
.map(|value| value.expose().to_owned())
|
||||
.or_else(|| std::env::var(name).ok())
|
||||
};
|
||||
let router = ModelRouter::from_config(config, &lookup);
|
||||
let settings = &config.litellm_settings;
|
||||
let http_settings = HttpSettings::from_layers([
|
||||
HttpSettingsLayer::from_environment(&lookup),
|
||||
|
|
@ -64,12 +65,7 @@ pub fn build_inference(config: &Config) -> Result<Arc<Gateway>, Error> {
|
|||
Arc::new(EnvironmentSecrets::python_compatible(client)),
|
||||
));
|
||||
let resources = CoreResources::new(pool);
|
||||
Ok(Arc::new(Gateway::new(
|
||||
resources,
|
||||
http,
|
||||
secrets,
|
||||
ModelRouter::from_model_list(&config.model_list),
|
||||
)?))
|
||||
Ok(Arc::new(Gateway::new(resources, http, secrets, router)?))
|
||||
}
|
||||
|
||||
pub async fn build_redis(config: &Config) -> Result<RedisClients, Error> {
|
||||
|
|
|
|||
|
|
@ -141,9 +141,13 @@ async fn provider_call(
|
|||
&env_lookup,
|
||||
)?;
|
||||
|
||||
let body = config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
let body = config.request_body(
|
||||
config.transform_anthropic_messages_request(
|
||||
trimmed,
|
||||
&MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
|
||||
)?,
|
||||
&validated.headers,
|
||||
shaping.settings.drop_params,
|
||||
)?;
|
||||
let environment = ValidatedEnvironment {
|
||||
headers: config.request_headers(
|
||||
|
|
@ -514,6 +518,84 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
fn vertex_call_requesting(
|
||||
shaping: MessagesShaping,
|
||||
requested: &str,
|
||||
drop_params: bool,
|
||||
) -> MessagesCall {
|
||||
MessagesCall {
|
||||
body: body(
|
||||
json!({"model": "claude-sonnet-4-5", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
|
||||
),
|
||||
api_key: None,
|
||||
api_base: Some("https://vertex.test".into()),
|
||||
custom_llm_provider: Some("vertex_ai".into()),
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: Some(Map::from_iter([
|
||||
("authorization".to_string(), json!("Bearer vertex-token")),
|
||||
("anthropic-version".to_string(), json!(requested)),
|
||||
])),
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
drop_params,
|
||||
..shaping.settings
|
||||
},
|
||||
..shaping
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn vertex_project(name: &str) -> Option<String> {
|
||||
(name == "VERTEXAI_PROJECT").then(|| "version-project".to_string())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_2("2026-10-21", false, "2026-10-21")]
|
||||
#[case::sdk_1("2023-06-01", false, "vertex-2023-10-16")]
|
||||
#[case::unknown_under_drop_params("latest", true, "vertex-2023-10-16")]
|
||||
#[tokio::test]
|
||||
async fn vertex_gets_the_requested_version_in_the_body_alone(
|
||||
shaping: MessagesShaping,
|
||||
#[case] requested: &str,
|
||||
#[case] drop_params: bool,
|
||||
#[case] body_version: &str,
|
||||
) {
|
||||
let call = provider_call_with_secrets(
|
||||
vertex_call_requesting(shaping, requested, drop_params),
|
||||
&vertex_project,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let version_headers: Vec<&str> = call
|
||||
.wire
|
||||
.headers
|
||||
.iter()
|
||||
.filter(|(name, _)| name.eq_ignore_ascii_case("anthropic-version"))
|
||||
.map(|(_, value)| value.as_str())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(version_headers, &call.wire.body["anthropic_version"]),
|
||||
(Vec::<&str>::new(), &json!(body_version))
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn vertex_refuses_an_unsupported_version_without_drop_params(shaping: MessagesShaping) {
|
||||
assert!(matches!(
|
||||
provider_call_with_secrets(
|
||||
vertex_call_requesting(shaping, "latest", false),
|
||||
&vertex_project
|
||||
)
|
||||
.await,
|
||||
Err(Error::Llm(litellm_llms::Error::InvalidRequest(
|
||||
litellm_llms::ErrorDetail::UnsupportedParameter { .. }
|
||||
)))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn missing_credentials_are_reported_before_the_request_is_transformed(
|
||||
|
|
|
|||
|
|
@ -77,6 +77,15 @@ pub trait BaseMessagesConfig: Sync {
|
|||
headers
|
||||
}
|
||||
|
||||
fn request_body(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
_headers: &Headers,
|
||||
_drop_params: bool,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// The JSON that goes on the wire, for a host whose body differs from the typed request:
|
||||
/// Python's configs `pop("model")` when the model is addressed by the URL.
|
||||
fn wire_body(&self, body: Value) -> Value {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# rules
|
||||
|
||||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, puts the Vertex `anthropic_version` in the body, and authenticates with a Google access token
|
||||
- Claude on Vertex AI: reuses `anthropic/messages` shaping, moves the model into the publisher URL, and authenticates with a Google access token
|
||||
- Vertex reads the API version only from the body's `anthropic_version` and answers 400 when an `anthropic-version` header names a different one, so the caller's header never reaches Vertex. A version Vertex accepts in the body is carried there, `2023-06-01`, `2023-01-01`, or no header get `vertex-2023-10-16`, and any other version drops under `drop_params` and is refused without it
|
||||
- Applies Vertex's beta policy and strips `output_config.effort` for models that do not accept it
|
||||
|
||||
# references
|
||||
|
|
|
|||
|
|
@ -3,10 +3,10 @@ use litellm_auth_gcp::{
|
|||
VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
|
||||
get_vertex_ai_project_from_credentials, secret_names,
|
||||
};
|
||||
use litellm_http::request::has_header;
|
||||
use litellm_http::request::{has_header, header_value, without_headers};
|
||||
use litellm_llms_types::{
|
||||
formats::messages::{MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet},
|
||||
providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, VERSION_HEADER},
|
||||
recognized::Recognized,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
|
|
@ -36,6 +36,8 @@ use litellm_llms_types::providers::vertex_ai::{
|
|||
};
|
||||
|
||||
const WEB_SEARCH_TOOL_PREFIX: &str = "web_search";
|
||||
const VERSIONS_SERVED_AS_VERTEX_DEFAULT: [&str; 3] = ["", "2023-01-01", "2023-06-01"];
|
||||
const VERSIONS_VERTEX_ACCEPTS_IN_BODY: [&str; 2] = [ANTHROPIC_VERSION, "2026-10-21"];
|
||||
|
||||
/// Claude on Vertex AI, Python's `VertexAIPartnerModelsAnthropicMessagesConfig`: the Anthropic
|
||||
/// payload addressed to a project and location, authenticated with a Google access token.
|
||||
|
|
@ -90,12 +92,6 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
.params
|
||||
.output_config
|
||||
.and_then(|config| sanitize_output_config(config, accepts_effort)),
|
||||
extra: request
|
||||
.params
|
||||
.extra
|
||||
.into_iter()
|
||||
.chain([("anthropic_version".to_string(), json!(ANTHROPIC_VERSION))])
|
||||
.collect(),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
|
|
@ -133,7 +129,35 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
|
|||
}
|
||||
|
||||
fn request_headers(&self, headers: Headers, request: &MessagesRequest) -> Headers {
|
||||
merge_beta_headers(headers, vertex_feature_betas(request))
|
||||
merge_beta_headers(
|
||||
without_headers(headers, &[VERSION_HEADER]),
|
||||
vertex_feature_betas(request),
|
||||
)
|
||||
}
|
||||
|
||||
fn request_body(
|
||||
&self,
|
||||
request: MessagesRequest,
|
||||
headers: &Headers,
|
||||
drop_params: bool,
|
||||
) -> Result<MessagesRequest, Error> {
|
||||
let version = vertex_version(
|
||||
&request.model,
|
||||
header_value(headers, VERSION_HEADER),
|
||||
drop_params,
|
||||
)?;
|
||||
Ok(MessagesRequest {
|
||||
params: MessagesOptionalParams {
|
||||
extra: request
|
||||
.params
|
||||
.extra
|
||||
.into_iter()
|
||||
.chain([("anthropic_version".to_string(), json!(version))])
|
||||
.collect(),
|
||||
..request.params
|
||||
},
|
||||
..request
|
||||
})
|
||||
}
|
||||
|
||||
/// Vertex addresses the model in the URL and rejects it in the body.
|
||||
|
|
@ -225,6 +249,26 @@ fn vertex_feature_betas(request: &MessagesRequest) -> BetaSet {
|
|||
)
|
||||
}
|
||||
|
||||
fn vertex_version<'a>(
|
||||
model: &str,
|
||||
requested: Option<&'a str>,
|
||||
drop_params: bool,
|
||||
) -> Result<&'a str, Error> {
|
||||
match requested.map(str::trim).unwrap_or_default() {
|
||||
version if VERSIONS_SERVED_AS_VERTEX_DEFAULT.contains(&version) => Ok(ANTHROPIC_VERSION),
|
||||
version if VERSIONS_VERTEX_ACCEPTS_IN_BODY.contains(&version) => Ok(version),
|
||||
_ if drop_params => Ok(ANTHROPIC_VERSION),
|
||||
version => Err(Error::InvalidRequest(
|
||||
crate::ErrorDetail::UnsupportedParameter {
|
||||
model: model.into(),
|
||||
param: VERSION_HEADER.into(),
|
||||
value: version.into(),
|
||||
hint: String::new(),
|
||||
},
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
|
@ -471,6 +515,28 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_spelling("anthropic-version")]
|
||||
#[case::any_case("Anthropic-Version")]
|
||||
fn a_forwarded_anthropic_version_never_reaches_vertex(#[case] name: &str) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG.request_headers(
|
||||
vec![
|
||||
(name.to_string(), "2026-10-21".to_string()),
|
||||
("x-request-id".to_string(), "req-1".to_string()),
|
||||
],
|
||||
&request,
|
||||
),
|
||||
vec![("x-request-id".to_string(), "req-1".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
fn transformed(value: Value, context: &MessagesTransformContext) -> Value {
|
||||
let request: MessagesRequest = serde_json::from_value(value).unwrap();
|
||||
serde_json::to_value(
|
||||
|
|
@ -482,16 +548,87 @@ mod tests {
|
|||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
|
||||
let body = transformed(
|
||||
json!({
|
||||
"model": "claude-sonnet-4-5@20250929",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}),
|
||||
&MessagesTransformContext::default(),
|
||||
#[case::sdk_2(&[("anthropic-version", "2026-10-21")], false, "2026-10-21")]
|
||||
#[case::any_case(&[("Anthropic-Version", "2026-10-21")], false, "2026-10-21")]
|
||||
#[case::vertex_name(&[("anthropic-version", "vertex-2023-10-16")], false, "vertex-2023-10-16")]
|
||||
#[case::sdk_1(&[("anthropic-version", "2023-06-01")], false, "vertex-2023-10-16")]
|
||||
#[case::first_version(&[("anthropic-version", "2023-01-01")], false, "vertex-2023-10-16")]
|
||||
#[case::blank(&[("anthropic-version", " ")], false, "vertex-2023-10-16")]
|
||||
#[case::no_header(&[], false, "vertex-2023-10-16")]
|
||||
#[case::unknown_under_drop_params(&[("anthropic-version", "latest")], true, "vertex-2023-10-16")]
|
||||
fn the_requested_version_goes_in_the_body(
|
||||
#[case] forwarded: &[(&str, &str)],
|
||||
#[case] drop_params: bool,
|
||||
#[case] body_version: &str,
|
||||
) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
let headers: Headers = forwarded
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect();
|
||||
let body = serde_json::to_value(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(request, &headers, drop_params)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["anthropic_version"], json!(body_version));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unreleased("2025-01-01")]
|
||||
#[case::alias("latest")]
|
||||
fn an_unsupported_version_is_refused_without_drop_params(#[case] requested: &str) {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}))
|
||||
.unwrap();
|
||||
let headers: Headers = vec![("anthropic-version".to_string(), requested.to_string())];
|
||||
let Error::InvalidRequest(detail) = VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(request, &headers, false)
|
||||
.unwrap_err()
|
||||
else {
|
||||
panic!("expected an invalid request");
|
||||
};
|
||||
assert_eq!(
|
||||
detail.to_string(),
|
||||
format!(
|
||||
"claude-sonnet-4-5 does not support anthropic-version={requested}. \
|
||||
To drop unsupported params, set `litellm.drop_params = True`."
|
||||
)
|
||||
);
|
||||
assert_eq!(body["anthropic_version"], json!("vertex-2023-10-16"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn the_body_carries_the_vertex_version_and_the_wire_drops_the_model() {
|
||||
let request: MessagesRequest = serde_json::from_value(json!({
|
||||
"model": "claude-sonnet-4-5@20250929",
|
||||
"max_tokens": 16,
|
||||
"messages": [{"role": "user", "content": "hi"}]
|
||||
}))
|
||||
.unwrap();
|
||||
let body = serde_json::to_value(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.request_body(
|
||||
VERTEX_ANTHROPIC_MESSAGES_CONFIG
|
||||
.transform_anthropic_messages_request(
|
||||
request,
|
||||
&MessagesTransformContext::default(),
|
||||
)
|
||||
.unwrap(),
|
||||
&Vec::new(),
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body["model"], json!("claude-sonnet-4-5@20250929"));
|
||||
let wire = VERTEX_ANTHROPIC_MESSAGES_CONFIG.wire_body(body);
|
||||
assert_eq!(wire.get("model"), None);
|
||||
|
|
|
|||
|
|
@ -1,3 +1,36 @@
|
|||
//! The model catalog: per-model pricing, context limits and capabilities.
|
||||
//!
|
||||
//! This is the Rust form of `model_prices_and_context_window.json`, the map Python reads as
|
||||
//! `litellm.model_cost` and through `litellm.get_model_info`. It answers "what does this model
|
||||
//! cost and what can it do"; it knows nothing about configured deployments. The router's view of
|
||||
//! deployments (groups, aliases, wildcard patterns, team mappings) is the router registry, defined
|
||||
//! in `litellm-router-core`, which reads this catalog through its dynamic config.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Catalog`: the parsed map. `Catalog::parse` takes the JSON body and a `Provenance` (where
|
||||
//! the body came from: the bundled copy in the repo or the remote copy, with revision and etag),
|
||||
//! and keeps every entry in file order with its raw fields.
|
||||
//! - `ModelEntry`: one model. `ModelEntry::fields` is the raw JSON object, so unknown fields are
|
||||
//! never lost; `ModelEntry::info` is the typed mirror `ModelInfo` (pricing in `pricing`,
|
||||
//! modalities and modes in `capabilities`).
|
||||
//! - Aliases: an entry may list `aliases`; each alias resolves to its canonical key. Problems
|
||||
//! found while indexing them (bad lists, collisions) are kept as `AliasIssue`s instead of
|
||||
//! failing the parse.
|
||||
//! - `FallbackGeneralizations` / `FallbackRule`: regex rules that map unknown model ids onto known
|
||||
//! families. They are data in the file, not entries.
|
||||
//! - Validation: `Catalog::validate` checks a catalog against `IntegrityLimits` (a remote copy that
|
||||
//! lost most of its models is refused), and `validate_registry` checks every entry against the
|
||||
//! checked-in contract.
|
||||
//!
|
||||
//! # Lookups
|
||||
//! `Catalog::lookup` resolves a key exactly first (entry or alias), then case-insensitively, and
|
||||
//! returns a `ModelMatch` with both the key that matched and the canonical key of the entry. Provider
|
||||
//! prefixes are not stripped here; callers decide which keys to try.
|
||||
//!
|
||||
//! # Readers
|
||||
//! Cost tracking, the router (through `RouterDynamicConfig`) and the gateway. Nothing in this crate
|
||||
//! performs I/O; loading and refreshing the file is the caller's job.
|
||||
|
||||
mod capabilities;
|
||||
mod catalog;
|
||||
mod error;
|
||||
|
|
|
|||
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
9
litellm-rust/crates/router-core/Cargo.toml
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
[package]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
|
||||
|
||||
[dependencies]
|
||||
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
63
litellm-rust/crates/router-core/src/lib.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
//! Routing for LiteLLM: decide which deployment serves a request, and drive retries, fallbacks
|
||||
//! and cooldowns around the attempts.
|
||||
//!
|
||||
//! This crate defines the boundary `litellm.Router` is moving behind. Today the Python router in
|
||||
//! `litellm/router.py` implements all of it; this crate is where the Rust core will implement the
|
||||
//! same interface. Until then it holds this overview and nothing else, so the interface is written
|
||||
//! down in code before either side changes.
|
||||
//!
|
||||
//! # The public surface
|
||||
//! Code outside the router uses `litellm.Router` through two kinds of members and nothing else:
|
||||
//! - The endpoint methods (`acompletion`, `aembedding`, `aresponses`, `aanthropic_messages`, the
|
||||
//! files, batches, assistants and vector store wrappers, 37 in all). Each is a thin wrapper over
|
||||
//! `call(op, request)`.
|
||||
//! - Eight entry points: `registry()`, `settings()`, `health()`, `select()`, `apply()`,
|
||||
//! `observe()`, `register_plugin()` and `retire()`.
|
||||
//!
|
||||
//! Every other member that outside code uses today has a named replacement among those. The old
|
||||
//! names stay available for about a month through a `DeprecatedRouterMembers` mixin, which
|
||||
//! implements them over the entry points and warns on use. A CI check
|
||||
//! (`tests/code_coverage_tests/router_surface_check.py`) keeps an allowlist of the members outside
|
||||
//! code touches; the list may only shrink.
|
||||
//!
|
||||
//! # Structure
|
||||
//! - `Registry`: the configured deployments, indexed by id, model group, alias, wildcard pattern
|
||||
//! and team. Immutable and versioned. Seven queries: `deployment(id)`, `deployments(filter)`,
|
||||
//! `names(filter)`, `resolve(name, scope)`, `group(name)`, `access_groups()` and
|
||||
//! `credentials(id)`. "Registry" is the router's view of what is configured; per-model pricing
|
||||
//! and capabilities are the model catalog (`litellm-model-catalog`), which the router reads
|
||||
//! through its dynamic config.
|
||||
//! - `RouterConfig` / `RouterDynamicConfig`: everything the router needs to know, passed in. The
|
||||
//! router never reads process globals. Static config (model list, strategy, retry, timeout and
|
||||
//! cooldown policy, Redis settings, plugins) is fixed for the router's life. Dynamic config
|
||||
//! (pricing, model aliases, fallbacks, retry counts, callbacks) is a versioned snapshot read
|
||||
//! once per call.
|
||||
//! - `Settings`: one typed, frozen view of the configuration a caller may read.
|
||||
//! - `Health`: cooldowns, blocked deployments and usage, kept in shared cache keys so Python and
|
||||
//! Rust workers see the same state. `report(id, outcome)` records outcomes the router did not
|
||||
//! see itself (a health check, an interceptor that marks a deployment failed).
|
||||
//! - `select(model, request)`: resolve a name, filter eligible deployments, pick one with the
|
||||
//! configured strategy, without calling it.
|
||||
//! - `apply(changes)`: one `ChangeSet` (upserts, deletes, settings), applied atomically, producing
|
||||
//! a new registry version.
|
||||
//! - `call(op, request)`: select, run the attempt, and retry or fall back until it succeeds or the
|
||||
//! policy is exhausted.
|
||||
//!
|
||||
//! # Execution
|
||||
//! Each attempt resolves an `Executor` after a deployment is picked. A Rust-native operation runs
|
||||
//! its `inference-*` route machine in-process; anything else goes through `Host::invoke`. A
|
||||
//! started attempt is never replayed on a different executor.
|
||||
//!
|
||||
//! # Hosts
|
||||
//! `Host` is what the embedding process provides: `invoke` for non-native operations, `select`
|
||||
//! for strategies and sub-routers not yet ported, `allow_fallback` for access and budget checks,
|
||||
//! `sleep`, `observe` for routing events, and `cache`, a DualCache-style store that holds all
|
||||
//! routing state under the same keys, encodings and TTLs the Python router uses. `PythonHost`
|
||||
//! serves them from the existing Python code when the router runs inside Python; `NativeHost`
|
||||
//! serves them in the Rust server, where an operation without a Rust implementation makes the
|
||||
//! config fail at build time, never partway through a request.
|
||||
//!
|
||||
//! # Plugins
|
||||
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
|
||||
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
|
||||
//! the current request), never the router itself.
|
||||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
litellm-config.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-inference.workspace = true
|
||||
litellm-inference-messages.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_model_list(&config.model_list)` maps configured public names to provider deployments. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
|
||||
`litellm-router` scaffolds the model-list setup and deployment lookup portion of Python's `litellm.Router`. `Router::from_config(&config, &env)` maps configured public names to provider deployments and reads `drop_params` from `litellm_settings` and each deployment, resolving an `os.environ/NAME` value through `env`. Programmatic callers can collect `(String, Deployment)` entries into a `Router`
|
||||
|
||||
Lookup is exact and returns `None` for an unknown name. This extraction preserves the gateway's existing behavior: the last entry wins when public names repeat. Multiple deployments per model group, routing strategies, retries, cooldowns, and fallbacks are not implemented yet
|
||||
|
||||
|
|
|
|||
|
|
@ -2,16 +2,33 @@ mod deployment;
|
|||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use litellm_config::Model;
|
||||
use litellm_config::{Config, Spelled};
|
||||
use litellm_core_utils::{serde_compat::parse_pydantic_bool, settings::Lookup};
|
||||
use litellm_inference_messages::{MessagesSettings, MessagesShaping};
|
||||
|
||||
pub use deployment::Deployment;
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Router(HashMap<String, Deployment>);
|
||||
|
||||
fn is_on(flag: Option<&Spelled<bool>>, env: &impl Lookup) -> bool {
|
||||
match flag {
|
||||
Some(Spelled::Value(on)) => *on,
|
||||
Some(Spelled::Text(text)) => match text.strip_prefix("os.environ/") {
|
||||
Some(name) => env
|
||||
.get(name)
|
||||
.is_some_and(|value| parse_pydantic_bool(&value) == Some(true)),
|
||||
None => parse_pydantic_bool(text) == Some(true),
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl Router {
|
||||
pub fn from_model_list(model_list: &[Model]) -> Self {
|
||||
model_list
|
||||
pub fn from_config(config: &Config, env: &impl Lookup) -> Self {
|
||||
let drops_params_globally = is_on(config.litellm_settings.drop_params.as_ref(), env);
|
||||
config
|
||||
.model_list
|
||||
.iter()
|
||||
.map(|model| {
|
||||
(
|
||||
|
|
@ -26,6 +43,14 @@ impl Router {
|
|||
api_base: model.litellm_params.api_base.clone(),
|
||||
custom_llm_provider: model.litellm_params.custom_llm_provider.clone(),
|
||||
litellm_params: model.litellm_params.clone(),
|
||||
shaping: MessagesShaping {
|
||||
settings: MessagesSettings {
|
||||
drop_params: drops_params_globally
|
||||
|| is_on(model.litellm_params.drop_params.as_ref(), env),
|
||||
..MessagesSettings::default()
|
||||
},
|
||||
..MessagesShaping::default()
|
||||
},
|
||||
..Deployment::default()
|
||||
},
|
||||
)
|
||||
|
|
@ -43,3 +68,60 @@ impl FromIterator<(String, Deployment)> for Router {
|
|||
Self(entries.into_iter().collect())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_config::Config;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::Router;
|
||||
|
||||
#[rstest]
|
||||
#[case::unset("", "", false)]
|
||||
#[case::deployment_flag("", "\n drop_params: true", true)]
|
||||
#[case::deployment_word("", "\n drop_params: \" True \"", true)]
|
||||
#[case::deployment_off("", "\n drop_params: false", false)]
|
||||
#[case::deployment_unknown_word("", "\n drop_params: maybe", false)]
|
||||
#[case::deployment_yes("", "\n drop_params: yes", true)]
|
||||
#[case::deployment_no("", "\n drop_params: \"no\"", false)]
|
||||
#[case::deployment_env_reference("", "\n drop_params: os.environ/DROP_PARAMS_FLAG", true)]
|
||||
#[case::deployment_unset_env_reference(
|
||||
"",
|
||||
"\n drop_params: os.environ/MISSING_FLAG",
|
||||
false
|
||||
)]
|
||||
#[case::global_env_reference(
|
||||
"litellm_settings:\n drop_params: os.environ/DROP_PARAMS_FLAG\n",
|
||||
"",
|
||||
true
|
||||
)]
|
||||
#[case::global_flag("litellm_settings:\n drop_params: true\n", "", true)]
|
||||
#[case::global_word("litellm_settings:\n drop_params: \"true\"\n", "", true)]
|
||||
#[case::global_off("litellm_settings:\n drop_params: false\n", "", false)]
|
||||
#[case::global_over_deployment_off(
|
||||
"litellm_settings:\n drop_params: true\n",
|
||||
"\n drop_params: false",
|
||||
true
|
||||
)]
|
||||
fn configured_drop_params_reach_the_deployment(
|
||||
#[case] settings: &str,
|
||||
#[case] deployment: &str,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let config = Config::from_yaml(&format!(
|
||||
"{settings}model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model{deployment}"
|
||||
))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG")
|
||||
.then(|| " Yes ".to_owned()))
|
||||
.get("public-model")
|
||||
.unwrap()
|
||||
.shaping
|
||||
.settings
|
||||
.drop_params,
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ fn configuration_preserves_deployment_parameters(#[case] parameters: &str) {
|
|||
"model_list:\n - model_name: public-model\n litellm_params:\n model: provider/model\n {parameters}"
|
||||
))
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
let deployment = router.get(&config.model_list[0].model_name).unwrap();
|
||||
let params = &config.model_list[0].litellm_params;
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ fn configuration_types_the_provider_params_of_a_deployment() {
|
|||
azure_ad_token: ignored-by-every-group",
|
||||
)
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
let params = &router.get("public-model").unwrap().litellm_params;
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -74,7 +74,7 @@ fn lookup_uses_public_names(#[case] name: &str, #[case] expected: Option<&str>)
|
|||
model: provider/b",
|
||||
)
|
||||
.unwrap();
|
||||
let router = Router::from_model_list(&config.model_list);
|
||||
let router = Router::from_config(&config, &|_: &str| None);
|
||||
|
||||
assert_eq!(router.get(name).map(|entry| entry.model.as_str()), expected);
|
||||
}
|
||||
|
|
@ -84,7 +84,7 @@ fn empty_configuration_has_no_deployment() {
|
|||
let config = Config::from_yaml("model_list: []").unwrap();
|
||||
|
||||
assert!(
|
||||
Router::from_model_list(&config.model_list)
|
||||
Router::from_config(&config, &|_: &str| None)
|
||||
.get("")
|
||||
.is_none()
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1336,7 +1336,6 @@ all_embedding_models = (
|
|||
)
|
||||
|
||||
####### IMAGE GENERATION MODELS ###################
|
||||
openai_image_generation_models = ["dall-e-2", "dall-e-3"]
|
||||
|
||||
####### VIDEO GENERATION MODELS ###################
|
||||
openai_video_generation_models = ["sora-2"]
|
||||
|
|
|
|||
|
|
@ -764,7 +764,6 @@ ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
|
|||
# Used for web search interception across providers
|
||||
LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search"
|
||||
|
||||
DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2"
|
||||
DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2"
|
||||
|
||||
DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)
|
||||
|
|
|
|||
|
|
@ -1592,14 +1592,6 @@ def completion_cost(
|
|||
for part in ws_tier_parts
|
||||
)
|
||||
|
||||
if (
|
||||
(call_type == "aimage_generation" or call_type == "image_generation")
|
||||
and model is not None
|
||||
and isinstance(model, str)
|
||||
and len(model) == 0
|
||||
and custom_llm_provider == "azure"
|
||||
):
|
||||
model = "dall-e-2" # for dall-e-2, azure expects an empty model name
|
||||
# Handle Inputs to completion_cost
|
||||
prompt_tokens = 0
|
||||
prompt_characters: int | None = None
|
||||
|
|
@ -1674,7 +1666,8 @@ def completion_cost(
|
|||
raw_candidates if custom_pricing is True else _priced_candidates_before_unpriced_rules(raw_candidates)
|
||||
)
|
||||
|
||||
for idx, (model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
for idx, (candidate_model, candidate_pricing_provider) in enumerate(potential_model_names):
|
||||
model = candidate_model
|
||||
try:
|
||||
if verbose_logger.isEnabledFor(logging.DEBUG):
|
||||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import litellm
|
|||
|
||||
# client is imported from litellm as it's a decorator
|
||||
from litellm import client
|
||||
from litellm.constants import DEFAULT_IMAGE_ENDPOINT_MODEL
|
||||
from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
|
||||
from litellm.exceptions import LiteLLMUnknownProvider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
|
|
@ -299,17 +298,14 @@ def image_generation(
|
|||
if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
|
||||
else None
|
||||
)
|
||||
dynamic_api_key: str | None = None
|
||||
if model is not None or custom_llm_provider is not None:
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
litellm_params=chatgpt_params,
|
||||
)
|
||||
else:
|
||||
model = "dall-e-2"
|
||||
custom_llm_provider = "openai" # default to dall-e-2 on openai
|
||||
if model is None and custom_llm_provider is None:
|
||||
raise litellm.BadRequestError(message="model is required", model="", llm_provider="")
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
api_base=api_base,
|
||||
litellm_params=chatgpt_params,
|
||||
)
|
||||
model_response.hidden_params["model"] = model
|
||||
openai_params: Final = [
|
||||
"user",
|
||||
|
|
@ -712,7 +708,7 @@ async def aimage_variation(*args, **kwargs) -> ImageResponse:
|
|||
@client
|
||||
def image_variation(
|
||||
image: FileTypes,
|
||||
model: str = "dall-e-2", # set to dall-e-2 by default - like OpenAI.
|
||||
model: str | None = None,
|
||||
n: int = 1,
|
||||
response_format: Literal["url", "b64_json"] = "url",
|
||||
size: str | None = None,
|
||||
|
|
@ -728,7 +724,7 @@ def image_variation(
|
|||
litellm_params: Final = get_litellm_params(**kwargs)
|
||||
# get the custom llm provider
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
model=model or "",
|
||||
custom_llm_provider=litellm_params.get("custom_llm_provider", None),
|
||||
api_base=litellm_params.get("api_base", None),
|
||||
api_key=litellm_params.get("api_key", None),
|
||||
|
|
@ -747,7 +743,7 @@ def image_variation(
|
|||
response: ImageResponse | None = None
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_model_info(
|
||||
model=model or "", # openai defaults to dall-e-2
|
||||
model=model,
|
||||
provider=llm_provider,
|
||||
)
|
||||
|
||||
|
|
@ -772,7 +768,7 @@ def image_variation(
|
|||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
model=model or None,
|
||||
image=image,
|
||||
timeout=litellm_params.get("timeout", None),
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -790,7 +786,7 @@ def image_variation(
|
|||
model_response=model_response,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
model=model,
|
||||
model=model or None,
|
||||
image=image,
|
||||
timeout=litellm_params.get("timeout", None) or DEFAULT_REQUEST_TIMEOUT,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -901,7 +897,7 @@ def image_edit(
|
|||
# get llm provider logic
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
model, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model or DEFAULT_IMAGE_ENDPOINT_MODEL,
|
||||
model=model or "",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -437,7 +437,6 @@ def get_llm_provider(
|
|||
model in litellm.open_ai_chat_completion_models
|
||||
or "ft:gpt-3.5-turbo" in model
|
||||
or "ft:gpt-4" in model # catches ft:gpt-4-0613, ft:gpt-4o
|
||||
or model in litellm.openai_image_generation_models
|
||||
or model.startswith("gpt-image")
|
||||
or model in litellm.openai_video_generation_models
|
||||
):
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ from pydantic import TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.batch_utils import InMemoryFile
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.router_utils.batch_utils import InMemoryFile
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from litellm.llms.anthropic.messages.native.transformation import (
|
|||
AnthropicMessagesConfig,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.native.utils import AnthropicMessagesRequestUtils, prepare_native_messages
|
||||
from litellm.router_utils.baseline_request import (
|
||||
from litellm.router_strategy.auto_router.baseline_request import (
|
||||
BASELINE_PARAMETERS,
|
||||
capture_baseline_parameters,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,13 +13,12 @@ from openai import (
|
|||
BadRequestError,
|
||||
OpenAI,
|
||||
)
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
|
||||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
|
||||
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
|
||||
from litellm.llms.custom_httpx.http_handler import (
|
||||
AsyncHTTPHandler,
|
||||
HTTPHandler,
|
||||
|
|
@ -34,7 +33,6 @@ from litellm.types.utils import (
|
|||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
convert_to_model_response_object,
|
||||
modify_url,
|
||||
)
|
||||
|
||||
from ...types.llms.openai import HttpxBinaryResponseContent
|
||||
|
|
@ -59,8 +57,6 @@ from .image_generation import (
|
|||
)
|
||||
from .image_generation.http_utils import azure_deployment_image_generation_json_body
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureOpenAIAssistantsAPIConfig:
|
||||
"""
|
||||
|
|
@ -860,11 +856,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers: dict,
|
||||
deployment_name: str | None = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Implemented for azure dall-e-2 image gen calls
|
||||
|
||||
Alternative to needing a custom transport implementation
|
||||
"""
|
||||
if client is None:
|
||||
_params: Final = {}
|
||||
if timeout is not None:
|
||||
|
|
@ -881,95 +872,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
else:
|
||||
async_handler = client
|
||||
|
||||
if (
|
||||
"images/generations" in api_base
|
||||
and api_version
|
||||
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
|
||||
"2023-06-01-preview",
|
||||
"2023-07-01-preview",
|
||||
"2023-08-01-preview",
|
||||
"2023-09-01-preview",
|
||||
"2023-10-01-preview",
|
||||
]
|
||||
): # CREATE + POLL for azure dall-e-2 calls
|
||||
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
|
||||
|
||||
data.pop(
|
||||
"model", None
|
||||
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
|
||||
response = await async_handler.post(
|
||||
url=api_base,
|
||||
data=json.dumps(data),
|
||||
headers=headers,
|
||||
)
|
||||
if "operation-location" in response.headers:
|
||||
operation_location_url: Final = response.headers["operation-location"]
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=response.text)
|
||||
# Reject polling URLs that don't share an origin with ``api_base``.
|
||||
# Without this an upstream-controlled or attacker-controlled
|
||||
# value would receive the operator's Azure API key in the
|
||||
# request headers below. VERIA-51.
|
||||
try:
|
||||
assert_same_origin(operation_location_url, api_base)
|
||||
except SSRFError as ssrf_err:
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message=f"Rejected polling URL: {ssrf_err}",
|
||||
)
|
||||
response = await async_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
await response.aread()
|
||||
|
||||
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
|
||||
start_time: Final = time.time()
|
||||
if "status" not in response.json():
|
||||
# Don't reflect the raw response body — when the polling
|
||||
# URL points at an internal JSON API (cloud metadata
|
||||
# service etc.) reflecting it here turns Blind SSRF into
|
||||
# Full-Read SSRF. VERIA-51.
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message="Polling response missing 'status' field",
|
||||
)
|
||||
while response.json()["status"] not in ["succeeded", "failed"]:
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
|
||||
|
||||
await asyncio.sleep(int(response.headers.get("retry-after") or 10))
|
||||
response = await async_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
await response.aread()
|
||||
|
||||
if response.json()["status"] == "failed":
|
||||
error_data: Final = response.json()
|
||||
# Preserve Azure error details (e.g. content_policy_violation,
|
||||
# inner_error, content_filter_results) as structured body so
|
||||
# exception_type() can route them correctly.
|
||||
_error_body: Final = error_data.get("error", error_data)
|
||||
_error_msg: Final = (
|
||||
_error_body.get("message", "Image generation failed")
|
||||
if isinstance(_error_body, dict)
|
||||
else json.dumps(error_data)
|
||||
)
|
||||
raise AzureOpenAIError(
|
||||
status_code=400,
|
||||
message=_error_msg,
|
||||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
content=json.dumps(result).encode("utf-8"),
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
|
||||
)
|
||||
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
|
||||
return await async_handler.post(
|
||||
url=api_base,
|
||||
|
|
@ -988,11 +890,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers: dict,
|
||||
deployment_name: str | None = None,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
Implemented for azure dall-e-2 image gen calls
|
||||
|
||||
Alternative to needing a custom transport implementation
|
||||
"""
|
||||
if client is None:
|
||||
_params: Final = {}
|
||||
if timeout is not None:
|
||||
|
|
@ -1006,87 +903,6 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
else:
|
||||
sync_handler = client
|
||||
|
||||
if (
|
||||
"images/generations" in api_base
|
||||
and api_version
|
||||
in [ # dall-e-3 starts from `2023-12-01-preview` so we should be able to avoid conflict
|
||||
"2023-06-01-preview",
|
||||
"2023-07-01-preview",
|
||||
"2023-08-01-preview",
|
||||
"2023-09-01-preview",
|
||||
"2023-10-01-preview",
|
||||
]
|
||||
): # CREATE + POLL for azure dall-e-2 calls
|
||||
api_base = modify_url(original_url=api_base, new_path="/openai/images/generations:submit")
|
||||
|
||||
data.pop(
|
||||
"model", None
|
||||
) # REMOVE 'model' from dall-e-2 arg https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#request-a-generated-image-dall-e-2-preview
|
||||
response = sync_handler.post(
|
||||
url=api_base,
|
||||
data=json.dumps(data),
|
||||
headers=headers,
|
||||
)
|
||||
if "operation-location" in response.headers:
|
||||
operation_location_url: Final = response.headers["operation-location"]
|
||||
else:
|
||||
raise AzureOpenAIError(status_code=500, message=response.text)
|
||||
try:
|
||||
assert_same_origin(operation_location_url, api_base)
|
||||
except SSRFError as ssrf_err:
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message=f"Rejected polling URL: {ssrf_err}",
|
||||
)
|
||||
response = sync_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
response.read()
|
||||
|
||||
timeout_secs: Final[int] = AZURE_OPERATION_POLLING_TIMEOUT
|
||||
start_time: Final = time.time()
|
||||
if "status" not in response.json():
|
||||
raise AzureOpenAIError(
|
||||
status_code=502,
|
||||
message="Polling response missing 'status' field",
|
||||
)
|
||||
while response.json()["status"] not in ["succeeded", "failed"]:
|
||||
if time.time() - start_time > timeout_secs:
|
||||
raise AzureOpenAIError(status_code=408, message="Operation polling timed out.")
|
||||
|
||||
time.sleep(int(response.headers.get("retry-after") or 10))
|
||||
response = sync_handler.get(
|
||||
url=operation_location_url,
|
||||
headers=headers,
|
||||
)
|
||||
response.read()
|
||||
|
||||
if response.json()["status"] == "failed":
|
||||
error_data: Final = response.json()
|
||||
# Preserve Azure error details (e.g. content_policy_violation,
|
||||
# inner_error, content_filter_results) as structured body so
|
||||
# exception_type() can route them correctly.
|
||||
_error_body: Final = error_data.get("error", error_data)
|
||||
_error_msg: Final = (
|
||||
_error_body.get("message", "Image generation failed")
|
||||
if isinstance(_error_body, dict)
|
||||
else json.dumps(error_data)
|
||||
)
|
||||
raise AzureOpenAIError(
|
||||
status_code=400,
|
||||
message=_error_msg,
|
||||
body=error_data,
|
||||
)
|
||||
|
||||
result: Final = _DECODED_JSON.validate_python(response.json()["result"])
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
headers=response.headers,
|
||||
content=json.dumps(result).encode("utf-8"),
|
||||
request=httpx.Request(method="POST", url="https://api.openai.com/v1"),
|
||||
)
|
||||
request_json: Final = azure_deployment_image_generation_json_body(api_base, data, deployment_name)
|
||||
return sync_handler.post(
|
||||
url=api_base,
|
||||
|
|
@ -1193,10 +1009,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
deployment_name=model,
|
||||
)
|
||||
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
|
||||
if provider_config is not None:
|
||||
return provider_config.transform_image_generation_response(
|
||||
model=data.get("model", "dall-e-2"),
|
||||
model=data.get("model", ""),
|
||||
raw_response=httpx_response,
|
||||
model_response=model_response or ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
|
|
@ -1329,10 +1145,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
|
|||
headers=request_headers,
|
||||
deployment_name=model,
|
||||
)
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))
|
||||
provider_config: Final = get_azure_image_generation_config(data.get("model", ""))
|
||||
if isinstance(provider_config, AzureFoundryMAIImageGenerationConfig):
|
||||
return provider_config.transform_image_generation_response(
|
||||
model=data.get("model", "dall-e-2"),
|
||||
model=data.get("model", ""),
|
||||
raw_response=httpx_response,
|
||||
model_response=model_response or ImageResponse(),
|
||||
logging_obj=logging_obj,
|
||||
|
|
|
|||
|
|
@ -4,14 +4,10 @@ from litellm.llms.base_llm.image_generation.transformation import (
|
|||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import AzureDallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import AzureDallE3ImageGenerationConfig
|
||||
from .gpt_transformation import AzureGPTImageGenerationConfig
|
||||
from .http_utils import azure_deployment_image_generation_json_body
|
||||
|
||||
__all__ = [
|
||||
"AzureDallE2ImageGenerationConfig",
|
||||
"AzureDallE3ImageGenerationConfig",
|
||||
"AzureGPTImageGenerationConfig",
|
||||
"azure_deployment_image_generation_json_body",
|
||||
]
|
||||
|
|
@ -21,11 +17,7 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
|||
model = model.lower()
|
||||
model = model.replace("-", "")
|
||||
model = model.replace("_", "")
|
||||
if model == "" or "dalle2" in model: # empty model is dall-e-2
|
||||
return AzureDallE2ImageGenerationConfig()
|
||||
elif "dalle3" in model:
|
||||
return AzureDallE3ImageGenerationConfig()
|
||||
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
return AzureFoundryMAIImageGenerationConfig()
|
||||
else:
|
||||
verbose_logger.debug(
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-2 image generation config
|
||||
"""
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-3 image generation config
|
||||
"""
|
||||
|
|
@ -3,15 +3,11 @@ from litellm.llms.base_llm.image_generation.transformation import (
|
|||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import AzureFoundryDallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import AzureFoundryDallE3ImageGenerationConfig
|
||||
from .flux_transformation import AzureFoundryFluxImageGenerationConfig
|
||||
from .gpt_transformation import AzureFoundryGPTImageGenerationConfig
|
||||
from .mai_transformation import AzureFoundryMAIImageGenerationConfig
|
||||
|
||||
__all__ = [
|
||||
"AzureFoundryDallE2ImageGenerationConfig",
|
||||
"AzureFoundryDallE3ImageGenerationConfig",
|
||||
"AzureFoundryFluxImageGenerationConfig",
|
||||
"AzureFoundryGPTImageGenerationConfig",
|
||||
"AzureFoundryMAIImageGenerationConfig",
|
||||
|
|
@ -22,11 +18,7 @@ def get_azure_ai_image_generation_config(model: str) -> BaseImageGenerationConfi
|
|||
model = model.lower()
|
||||
model = model.replace("-", "")
|
||||
model = model.replace("_", "")
|
||||
if model == "" or "dalle2" in model: # empty model is dall-e-2
|
||||
return AzureFoundryDallE2ImageGenerationConfig()
|
||||
elif "dalle3" in model:
|
||||
return AzureFoundryDallE3ImageGenerationConfig()
|
||||
elif AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
if AzureFoundryMAIImageGenerationConfig.is_mai_model(model):
|
||||
return AzureFoundryMAIImageGenerationConfig()
|
||||
elif "flux" in model:
|
||||
return AzureFoundryFluxImageGenerationConfig()
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE2ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE2ImageGenerationConfig(DallE2ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-2 image generation config
|
||||
"""
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
from litellm.llms.openai.image_generation import DallE3ImageGenerationConfig
|
||||
|
||||
|
||||
class AzureFoundryDallE3ImageGenerationConfig(DallE3ImageGenerationConfig):
|
||||
"""
|
||||
Azure dall-e-3 image generation config
|
||||
"""
|
||||
|
|
@ -561,11 +561,8 @@ class BaseLLMAIOHTTPHandler:
|
|||
organization: str | None = None,
|
||||
headers: dict | None = None,
|
||||
) -> ImageResponse:
|
||||
if model is None:
|
||||
raise ValueError("model is required for non-openai image variations")
|
||||
|
||||
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
|
||||
model=model, # openai defaults to dall-e-2
|
||||
model=model or "",
|
||||
provider=LlmProviders(custom_llm_provider),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,31 +1,5 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
|
||||
|
||||
from .dalle2_transformation import DallE2ImageEditConfig
|
||||
from .transformation import OpenAIImageEditConfig
|
||||
|
||||
__all__ = [
|
||||
"DallE2ImageEditConfig",
|
||||
"OpenAIImageEditConfig",
|
||||
"get_openai_image_edit_config",
|
||||
]
|
||||
|
||||
|
||||
def get_openai_image_edit_config(model: str) -> BaseImageEditConfig:
|
||||
"""
|
||||
Get the appropriate OpenAI image edit config based on the model.
|
||||
|
||||
Args:
|
||||
model: The model name (e.g., "dall-e-2", "gpt-image-1")
|
||||
|
||||
Returns:
|
||||
The appropriate config instance for the model
|
||||
"""
|
||||
model_normalized: Final = model.lower().replace("-", "").replace("_", "")
|
||||
|
||||
if model_normalized == "dalle2":
|
||||
return DallE2ImageEditConfig()
|
||||
else:
|
||||
# Default to standard OpenAI config for gpt-image-1 and other models
|
||||
return OpenAIImageEditConfig()
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
from io import BufferedReader
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from httpx._types import RequestFiles
|
||||
|
||||
import litellm
|
||||
from litellm.images.utils import ImageEditRequestUtils
|
||||
from litellm.types.images.main import ImageEditRequestParams
|
||||
from litellm.types.llms.openai import FileTypes
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from .transformation import OpenAIImageEditConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
|
||||
|
||||
LiteLLMLoggingObj = _LiteLLMLoggingObj
|
||||
else:
|
||||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
class DallE2ImageEditConfig(OpenAIImageEditConfig):
|
||||
"""
|
||||
DALL-E-2 specific configuration for image edit API.
|
||||
|
||||
DALL-E-2 only supports editing a single image (not an array).
|
||||
Uses "image" field name instead of "image[]".
|
||||
"""
|
||||
|
||||
def transform_image_edit_request(
|
||||
self,
|
||||
model: str,
|
||||
prompt: str | None,
|
||||
image: FileTypes | None,
|
||||
image_edit_optional_request_params: dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
) -> tuple[dict, RequestFiles]:
|
||||
"""
|
||||
Transform image edit request for DALL-E-2.
|
||||
|
||||
DALL-E-2 only accepts a single image with field name "image" (not "image[]").
|
||||
"""
|
||||
request_params: Final = {
|
||||
"model": model,
|
||||
**image_edit_optional_request_params,
|
||||
}
|
||||
if image is not None:
|
||||
request_params["image"] = image
|
||||
if prompt is not None:
|
||||
request_params["prompt"] = prompt
|
||||
|
||||
request: Final = ImageEditRequestParams(**request_params)
|
||||
request_dict: Final = cast(dict, request)
|
||||
|
||||
#########################################################
|
||||
# Separate images and masks as `files` and send other parameters as `data`
|
||||
#########################################################
|
||||
_image_list: Final = request_dict.get("image")
|
||||
_mask = request_dict.get("mask")
|
||||
data_without_files: Final = {k: v for k, v in request_dict.items() if k not in ["image", "mask"]}
|
||||
files_list: Final[list[tuple[str, Any]]] = []
|
||||
|
||||
# Handle image parameter - DALL-E-2 only supports single image
|
||||
if _image_list is not None:
|
||||
image_list: Final = [_image_list] if not isinstance(_image_list, list) else _image_list
|
||||
|
||||
# Validate only one image is provided
|
||||
if len(image_list) > 1:
|
||||
raise litellm.BadRequestError(
|
||||
message="DALL-E-2 only supports editing a single image. Please provide one image.",
|
||||
model=model,
|
||||
llm_provider="openai",
|
||||
)
|
||||
|
||||
# Use "image" field name (singular) for DALL-E-2
|
||||
for _image in image_list:
|
||||
if _image is not None:
|
||||
self._add_image_to_files(
|
||||
files_list=files_list,
|
||||
image=_image,
|
||||
field_name="image",
|
||||
)
|
||||
|
||||
# Handle mask parameter if provided
|
||||
if _mask is not None:
|
||||
# Handle case where mask can be a list (extract first mask)
|
||||
if isinstance(_mask, list):
|
||||
_mask = _mask[0] if _mask else None
|
||||
|
||||
if _mask is not None:
|
||||
mask_content_type: Final[str] = ImageEditRequestUtils.get_image_content_type(_mask)
|
||||
if isinstance(_mask, BufferedReader):
|
||||
files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
|
||||
else:
|
||||
files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
|
||||
|
||||
return data_without_files, files_list
|
||||
|
|
@ -96,7 +96,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
|
|||
"""
|
||||
# Build request params, only including non-None values
|
||||
request_params: Final = {
|
||||
"model": model,
|
||||
**({"model": model} if model else {}),
|
||||
**image_edit_optional_request_params,
|
||||
}
|
||||
if image is not None:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
||||
from .dall_e_2_transformation import DallE2ImageGenerationConfig
|
||||
from .dall_e_3_transformation import DallE3ImageGenerationConfig
|
||||
from .gpt_transformation import GPTImageGenerationConfig
|
||||
from .guardrail_translation import (
|
||||
OpenAIImageGenerationHandler,
|
||||
|
|
@ -11,18 +5,7 @@ from .guardrail_translation import (
|
|||
)
|
||||
|
||||
__all__ = [
|
||||
"DallE2ImageGenerationConfig",
|
||||
"DallE3ImageGenerationConfig",
|
||||
"GPTImageGenerationConfig",
|
||||
"OpenAIImageGenerationHandler",
|
||||
"guardrail_translation_mappings",
|
||||
]
|
||||
|
||||
|
||||
def get_openai_image_generation_config(model: str) -> BaseImageGenerationConfig:
|
||||
if model.startswith("dall-e-2") or model == "": # empty model is dall-e-2
|
||||
return DallE2ImageGenerationConfig()
|
||||
elif model.startswith("dall-e-3"):
|
||||
return DallE3ImageGenerationConfig()
|
||||
else:
|
||||
return GPTImageGenerationConfig()
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class DallE2ImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
OpenAI dall-e-2 image generation config
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "response_format", "quality", "size", "user"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
for k in non_default_params:
|
||||
if k not in optional_params:
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
response: Final = raw_response.json()
|
||||
|
||||
stringified_response: Final = response
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
original_response=stringified_response,
|
||||
)
|
||||
image_response: Final[ImageResponse] = convert_to_model_response_object(
|
||||
response_object=stringified_response,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
|
||||
# set optional params
|
||||
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
|
||||
image_response.quality = optional_params.get("quality", "standard") # always standard for dall-e-2
|
||||
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-2
|
||||
|
||||
return image_response
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
|
||||
from litellm.types.utils import ImageResponse
|
||||
from litellm.utils import convert_to_model_response_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
||||
|
||||
class DallE3ImageGenerationConfig(BaseImageGenerationConfig):
|
||||
"""
|
||||
OpenAI dall-e-3 image generation config
|
||||
"""
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
return ["n", "response_format", "quality", "size", "user", "style"]
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
) -> dict:
|
||||
supported_params: Final = self.get_supported_openai_params(model)
|
||||
for k in non_default_params:
|
||||
if k not in optional_params:
|
||||
if k in supported_params:
|
||||
optional_params[k] = non_default_params[k]
|
||||
elif drop_params:
|
||||
pass
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Parameter {k} is not supported for model {model}. Supported parameters are {supported_params}. Set drop_params=True to drop unsupported parameters."
|
||||
)
|
||||
|
||||
return optional_params
|
||||
|
||||
def transform_image_generation_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
model_response: ImageResponse,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
request_data: dict,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
encoding: "Tokenizer | None",
|
||||
api_key: str | None = None,
|
||||
json_mode: bool | None = None,
|
||||
) -> ImageResponse:
|
||||
response: Final = raw_response.json()
|
||||
|
||||
stringified_response: Final = response
|
||||
## LOGGING
|
||||
logging_obj.post_call(
|
||||
input=request_data.get("prompt", ""),
|
||||
api_key=api_key,
|
||||
additional_args={"complete_input_dict": request_data},
|
||||
original_response=stringified_response,
|
||||
)
|
||||
image_response: Final[ImageResponse] = convert_to_model_response_object(
|
||||
response_object=stringified_response,
|
||||
model_response_object=model_response,
|
||||
response_type="image_generation",
|
||||
)
|
||||
|
||||
# set optional params
|
||||
image_response.size = optional_params.get("size", "1024x1024") # default is always 1024x1024
|
||||
image_response.quality = optional_params.get("quality", "hd") # always hd for dall-e-3
|
||||
image_response.output_format = optional_params.get("output_format", "png") # always png for dall-e-3
|
||||
|
||||
return image_response
|
||||
|
|
@ -151,7 +151,7 @@ class OpenAIImageVariationsHandler:
|
|||
) -> ImageResponse:
|
||||
try:
|
||||
provider_config: Final = ProviderConfigManager.get_provider_image_variation_config(
|
||||
model=model or "", # openai defaults to dall-e-2
|
||||
model=model or "",
|
||||
provider=LlmProviders.OPENAI,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from litellm.exceptions import UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.anthropic.messages.native.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -17,6 +19,41 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from ....vertex_llm_base import VertexBase
|
||||
from ..output_params_utils import sanitize_vertex_anthropic_output_params
|
||||
|
||||
_VERSION_HEADER_VERTEX_READS_FROM_BODY: Final = "anthropic-version"
|
||||
_VERTEX_DEFAULT_VERSION: Final = "vertex-2023-10-16"
|
||||
_VERSIONS_SERVED_AS_VERTEX_DEFAULT: Final = frozenset({"", "2023-01-01", "2023-06-01"})
|
||||
_VERSIONS_VERTEX_ACCEPTS_IN_BODY: Final = frozenset({_VERTEX_DEFAULT_VERSION, "2026-10-21"})
|
||||
|
||||
|
||||
def _requested_version(headers: Mapping[str, object]) -> str:
|
||||
requested: Final = next(
|
||||
(value for name, value in headers.items() if name.lower() == _VERSION_HEADER_VERTEX_READS_FROM_BODY),
|
||||
"",
|
||||
)
|
||||
return requested.strip() if isinstance(requested, str) else ""
|
||||
|
||||
|
||||
def _vertex_version(headers: Mapping[str, object], model: str, drop_params: bool) -> str | UnsupportedParamsError:
|
||||
requested: Final = _requested_version(headers)
|
||||
if requested in _VERSIONS_SERVED_AS_VERTEX_DEFAULT:
|
||||
return _VERTEX_DEFAULT_VERSION
|
||||
if requested in _VERSIONS_VERTEX_ACCEPTS_IN_BODY:
|
||||
return requested
|
||||
if drop_params:
|
||||
return _VERTEX_DEFAULT_VERSION
|
||||
return UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {_VERSION_HEADER_VERTEX_READS_FROM_BODY}={requested}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
llm_provider="vertex_ai",
|
||||
model=model,
|
||||
)
|
||||
|
||||
|
||||
def _without_version_header(headers: Mapping[str, str]) -> dict[str, str]: # mutable-ok: the caller sends these headers
|
||||
return {name: value for name, value in headers.items() if name.lower() != _VERSION_HEADER_VERTEX_READS_FROM_BODY}
|
||||
|
||||
|
||||
class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
|
||||
@property
|
||||
|
|
@ -198,7 +235,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
messages: list[dict],
|
||||
anthropic_messages_optional_request_params: dict,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
headers: dict,
|
||||
headers: dict[str, object], # mutable-ok: mirrors the provider transform contract
|
||||
) -> dict:
|
||||
anthropic_messages_request: Final = super().transform_anthropic_messages_request(
|
||||
model=model,
|
||||
|
|
@ -212,10 +249,28 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
|
|||
|
||||
self._remove_scope_from_cache_control(anthropic_messages_request)
|
||||
|
||||
anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16"
|
||||
version: Final = _vertex_version(
|
||||
headers, model=model, drop_params=drops_unsupported_params(litellm_params.drop_params)
|
||||
)
|
||||
if isinstance(version, UnsupportedParamsError):
|
||||
raise version
|
||||
anthropic_messages_request["anthropic_version"] = version
|
||||
|
||||
anthropic_messages_request.pop("model", None) # do not pass model in request body to vertex ai
|
||||
|
||||
sanitize_vertex_anthropic_output_params(anthropic_messages_request, model)
|
||||
|
||||
return anthropic_messages_request
|
||||
|
||||
def sign_request(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
optional_params: Mapping[str, object],
|
||||
request_data: Mapping[str, object],
|
||||
api_base: str,
|
||||
api_key: str | None = None,
|
||||
model: str | None = None,
|
||||
stream: bool | None = None,
|
||||
fake_stream: bool | None = None,
|
||||
) -> tuple[dict[str, str], bytes | None]: # mutable-ok: the caller sends these headers
|
||||
return _without_version_header(headers), None
|
||||
|
|
|
|||
|
|
@ -139,8 +139,8 @@ from litellm.proxy.utils import ( # noqa: F401 # legacy module exports
|
|||
)
|
||||
from litellm.responses.streaming_iterator import partial_responses_incomplete_event_on_disconnect, spawn_logging_task
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
|
||||
from litellm.types.extraction import ExtractionResponse
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from typing import TYPE_CHECKING, Final
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
|
||||
from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from litellm.constants import (
|
|||
HEALTH_CHECK_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependency,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
_effective_model_info, # pyright: ignore[reportPrivateUsage] # existing deployment-price owner
|
||||
_proxy_llm_router, # pyright: ignore[reportPrivateUsage] # existing optional proxy-router owner
|
||||
)
|
||||
from litellm.router_strategy.auto_router.baseline_request import baseline_request
|
||||
from litellm.router_strategy.complexity_router.context_compaction import compaction_applied
|
||||
from litellm.router_utils.baseline_request import baseline_request
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
from litellm.types.utils import CallTypes, ModelInfo, Usage
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import Span
|
||||
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.llms.bedrock.common_utils import get_bedrock_base_model, get_bedrock_cross_region_inference_regions
|
||||
|
|
@ -30,7 +31,6 @@ from litellm.proxy.hooks.model_budget_shared_windows import (
|
|||
SharedWindows,
|
||||
)
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting, budget_window_seconds
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import BudgetConfig, LlmProviders, StandardLoggingPayload
|
||||
|
||||
|
|
|
|||
|
|
@ -3290,7 +3290,7 @@ def _per_user_credential_names_for_groups(
|
|||
GITHUB_COPILOT_AUTH_TYPE_KEY,
|
||||
GITHUB_COPILOT_PER_USER_AUTH_TYPE,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import classify_strategy_router_model
|
||||
|
||||
names: Final[list[str]] = [] # mutable-ok: accumulates one name per per-user deployment
|
||||
for group in model_groups:
|
||||
|
|
|
|||
|
|
@ -60,16 +60,16 @@ from litellm.repositories.table_repositories import PrismaTableRepository
|
|||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
StrategyRouterDependencyRole,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.request_models import (
|
||||
AutoRouterRoutingTestRequest,
|
||||
RequestComplexityRouterConfig,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
StrategyRouterDependencyRole,
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
SHADOW_EVAL_TURN_VALVE,
|
||||
|
|
@ -411,7 +411,7 @@ async def validate_complexity_router_config(
|
|||
auto_router_environment_reference_error(data.complexity_router_config, user_api_key_dict)
|
||||
)
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
validate_complexity_router_config_write,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,23 @@ from litellm.repositories.project_repository import ProjectRepository
|
|||
from litellm.repositories.table_repositories import ModelTableRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
is_complexity_router_model,
|
||||
validate_complexity_router_config_placement,
|
||||
validate_complexity_router_config_write,
|
||||
validate_strategy_router_model_write,
|
||||
)
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router import (
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
ClassificationRubric,
|
||||
|
|
@ -144,20 +161,6 @@ from litellm.router_strategy.complexity_router import (
|
|||
normalize_classification_prompt,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import resolve_complexity_router_config_write
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
is_complexity_router_model,
|
||||
validate_complexity_router_config_placement,
|
||||
validate_complexity_router_config_write,
|
||||
validate_strategy_router_model_write,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import is_mutable_tuned_candidate, tuning_quota_violation
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.llms.bedrock import AwsSessionTag
|
||||
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
|
||||
|
|
|
|||
|
|
@ -7,14 +7,14 @@ from typing import Final
|
|||
from pydantic import Json, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capabilities_of,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
mutable_tuned_identities,
|
||||
tuning_quota_violation,
|
||||
|
|
|
|||
|
|
@ -34,13 +34,16 @@ from litellm.repositories.prisma_protocols import DatabaseClient
|
|||
from litellm.repositories.project_repository import ProjectRepository
|
||||
from litellm.repositories.table_repositories import TeamMembershipRepository
|
||||
from litellm.router import Router
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
classify_strategy_router_model,
|
||||
strategy_router_dependencies,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
ComplexityRouterConfigWrite,
|
||||
normalize_classifier_config_aliases,
|
||||
resolve_complexity_router_config_write,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
|
||||
from litellm.router_utils.auto_router_model_naming import classify_strategy_router_model, strategy_router_dependencies
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.router import Deployment, updateDeployment
|
||||
|
||||
|
|
|
|||
|
|
@ -168,12 +168,7 @@ from litellm.proxy.common_utils.token_count_settings import (
|
|||
)
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
capability_limit_violation,
|
||||
|
|
@ -181,12 +176,17 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
count_capability_routers,
|
||||
validate_complexity_router_config_placement,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
TUNING_BASELINE_PARAM_NAME,
|
||||
mutable_tuned_identities,
|
||||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
get_hidden_params_dict,
|
||||
)
|
||||
from litellm.router_utils.common_utils import resolve_model_group_alias
|
||||
from litellm.router_utils.routing_groups import parse_routing_groups
|
||||
from litellm.types.caching import RedisPipelineIncrementOperation
|
||||
|
|
|
|||
|
|
@ -7,6 +7,29 @@
|
|||
#
|
||||
# Thank you ! We ❤️ you! - Krrish & Ishaan
|
||||
|
||||
"""
|
||||
litellm.Router: load balancing, retries, fallbacks and cooldowns across deployments.
|
||||
|
||||
Public surface. The router is moving behind a narrow interface; code outside
|
||||
the router should end up using a Router through two kinds of members only:
|
||||
|
||||
- endpoint methods (acompletion, aembedding, aresponses, aanthropic_messages,
|
||||
the files, batches, assistants and vector store wrappers, ...), each a thin
|
||||
wrapper around one call;
|
||||
- entry points (being added) that replace direct access to internals: registry() for
|
||||
deployment lookups, settings() for configuration reads, health() for
|
||||
cooldowns, blocks and usage, select() to pick a deployment without calling
|
||||
it, apply() for runtime changes, observe() for routing events,
|
||||
register_plugin() for strategies and sub-routers, and retire() to release
|
||||
the router.
|
||||
|
||||
Everything else on the class is internal. The members outside code uses today
|
||||
are listed in tests/code_coverage_tests/router_surface_allowlist.json, and a CI
|
||||
check (router_surface_check.py next to it) fails on any new outside use, so the
|
||||
list only shrinks while callers move to the entry points. The same interface
|
||||
is documented for the Rust core in litellm-rust/crates/router-core.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import copy
|
||||
|
|
@ -85,6 +108,15 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.integrations.otel.routing import routing_decision_attributes
|
||||
from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
deployment_supported_endpoints,
|
||||
get_router_metadata_variable_name,
|
||||
is_batch_retrieve_call_type,
|
||||
replace_model_in_jsonl,
|
||||
should_replace_model_in_jsonl,
|
||||
without_deployments_that_cannot_batch,
|
||||
)
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
coerce_token_limit,
|
||||
|
|
@ -148,6 +180,15 @@ from litellm.llms.openai_like.model_info import (
|
|||
get_openai_compatible_model_info,
|
||||
)
|
||||
from litellm.responses.bridge_check import keep_registered_mode
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
claimed_capabilities,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.context_compaction import (
|
||||
|
|
@ -187,24 +228,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
replace_complexity_router_headers,
|
||||
response_total_token_count,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
AUTO_ROUTER_MODEL_PREFIX,
|
||||
STRATEGY_ROUTER_PARAM_FIELDS,
|
||||
GatedAutoRouterCapability,
|
||||
capability_limit_violation,
|
||||
claimed_capabilities,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_utils.batch_utils import (
|
||||
batch_create_routing,
|
||||
deployment_supported_endpoints,
|
||||
get_router_metadata_variable_name,
|
||||
is_batch_retrieve_call_type,
|
||||
replace_model_in_jsonl,
|
||||
should_replace_model_in_jsonl,
|
||||
without_deployments_that_cannot_batch,
|
||||
)
|
||||
from litellm.router_utils.client_initalization_utils import InitalizeCachedClient, MaxParallelRequestsLimit
|
||||
from litellm.router_utils.clientside_credential_handler import (
|
||||
get_dynamic_litellm_params,
|
||||
|
|
@ -16271,7 +16294,7 @@ class Router:
|
|||
to the deployment that actually served the request. Every attempt therefore
|
||||
writes or clears, never just writes.
|
||||
"""
|
||||
from litellm.router_utils.baseline_request import capture_baseline_parameters
|
||||
from litellm.router_strategy.auto_router.baseline_request import capture_baseline_parameters
|
||||
from litellm.types.router import BaselineRouteStamp
|
||||
|
||||
phase_attributes(routing_decision_attributes(routing_decision))
|
||||
|
|
|
|||
|
|
@ -36,12 +36,12 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import RedisCache, RedisPipelineIncrementOperation, log_redis_failure
|
||||
from litellm.integrations.custom_logger import CustomLogger, Span
|
||||
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_metadata_variable_name_from_kwargs,
|
||||
)
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
|
||||
from litellm.router_strategy.tag_based_routing import get_tags_from_request_kwargs
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key, batch_id_to_charge_once
|
||||
from litellm.router_utils.cooldown_callbacks import (
|
||||
get_prometheus_logger_from_callbacks,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from litellm._logging import verbose_router_logger
|
|||
from litellm.caching.caching import DualCache
|
||||
from litellm.caching.redis_cache import log_redis_failure
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
|
||||
IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
|
||||
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@ from litellm import ModelResponse, token_counter, verbose_logger
|
|||
from litellm._internal_context import with_service_target
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase
|
||||
from litellm.utils import print_verbose
|
||||
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ from litellm._internal_context import with_service_target
|
|||
from litellm._logging import verbose_logger, verbose_router_logger
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.router import RouterErrors
|
||||
from litellm.types.utils import LiteLLMPydanticObjectBase, StandardLoggingPayload
|
||||
from litellm.utils import get_utc_datetime, print_verbose
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from pydantic import TypeAdapter
|
|||
import litellm
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
|
||||
from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy
|
||||
from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
|
|
@ -22,7 +23,6 @@ from litellm.router_utils.add_retry_fallback_headers import (
|
|||
add_fallback_headers_to_response,
|
||||
get_fallback_error_info,
|
||||
)
|
||||
from litellm.router_utils.batch_utils import get_router_metadata_variable_name
|
||||
from litellm.router_utils.cooldown_handlers import (
|
||||
_first_present, # pyright: ignore[reportPrivateUsage] - shared internal helper, used across router_utils
|
||||
cast_exception_status_to_int,
|
||||
|
|
|
|||
|
|
@ -7664,7 +7664,6 @@ def validate_environment(
|
|||
model in litellm.open_ai_chat_completion_models
|
||||
or model in litellm.open_ai_text_completion_models
|
||||
or model in litellm.open_ai_embedding_models
|
||||
or model in litellm.openai_image_generation_models
|
||||
or model.startswith("gpt-image")
|
||||
):
|
||||
if "OPENAI_API_KEY" in os.environ:
|
||||
|
|
@ -10311,11 +10310,9 @@ class ProviderConfigManager:
|
|||
provider: LlmProviders,
|
||||
) -> BaseImageGenerationConfig | None:
|
||||
if LlmProviders.OPENAI == provider:
|
||||
from litellm.llms.openai.image_generation import (
|
||||
get_openai_image_generation_config,
|
||||
)
|
||||
from litellm.llms.openai.image_generation import GPTImageGenerationConfig
|
||||
|
||||
return get_openai_image_generation_config(model)
|
||||
return GPTImageGenerationConfig()
|
||||
elif LlmProviders.AZURE == provider:
|
||||
from litellm.llms.azure.image_generation import (
|
||||
get_azure_image_generation_config,
|
||||
|
|
@ -10546,9 +10543,9 @@ class ProviderConfigManager:
|
|||
provider: LlmProviders,
|
||||
) -> BaseImageEditConfig | None:
|
||||
if LlmProviders.OPENAI == provider:
|
||||
from litellm.llms.openai.image_edit import get_openai_image_edit_config
|
||||
from litellm.llms.openai.image_edit import OpenAIImageEditConfig
|
||||
|
||||
return get_openai_image_edit_config(model=model)
|
||||
return OpenAIImageEditConfig()
|
||||
elif LlmProviders.HOSTED_VLLM == provider:
|
||||
from litellm.llms.hosted_vllm.image_edit import get_hosted_vllm_image_edit_config
|
||||
|
||||
|
|
|
|||
|
|
@ -187,13 +187,13 @@ ignored_function_names = [
|
|||
def main():
|
||||
router_file = [
|
||||
"./litellm/router.py",
|
||||
"./litellm/router_utils/batch_utils.py",
|
||||
"./litellm/litellm_core_utils/batch_utils.py",
|
||||
"./litellm/router_utils/pattern_match_deployments.py",
|
||||
]
|
||||
# router_file = [
|
||||
# "../../litellm/router.py",
|
||||
# "../../litellm/router_utils/pattern_match_deployments.py",
|
||||
# "../../litellm/router_utils/batch_utils.py",
|
||||
# "../../litellm/litellm_core_utils/batch_utils.py",
|
||||
# ] ## LOCAL TESTING
|
||||
tests_dir = "./tests/" # Update this path if your tests directory is located elsewhere
|
||||
# tests_dir = "../../tests/" # LOCAL TESTING
|
||||
|
|
|
|||
211
tests/code_coverage_tests/router_surface_allowlist.json
Normal file
211
tests/code_coverage_tests/router_surface_allowlist.json
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
{
|
||||
"_doc": "Router members and litellm.router_utils modules used outside the router. Shrink-only: see tests/code_coverage_tests/router_surface_check.py.",
|
||||
"members_outside": [
|
||||
"_get_client",
|
||||
"_zero_cost_cache",
|
||||
"a_add_message",
|
||||
"aadapter_completion",
|
||||
"aanthropic_messages",
|
||||
"abatch_completion",
|
||||
"abatch_completion_fastest_response",
|
||||
"acancel_batch",
|
||||
"acancel_fine_tuning_job",
|
||||
"acompletion",
|
||||
"acreate_assistants",
|
||||
"acreate_batch",
|
||||
"acreate_file",
|
||||
"acreate_fine_tuning_job",
|
||||
"acreate_thread",
|
||||
"adaptive_routers",
|
||||
"adecisions",
|
||||
"adelete_assistant",
|
||||
"aembedding",
|
||||
"afile_content",
|
||||
"afile_delete",
|
||||
"aget_assistants",
|
||||
"aget_messages",
|
||||
"aget_responses",
|
||||
"aget_thread",
|
||||
"alist_batches",
|
||||
"alist_fine_tuning_jobs",
|
||||
"allm_passthrough_route",
|
||||
"amoderation",
|
||||
"aocr",
|
||||
"are_all_deployments_blocked",
|
||||
"arefresh_model_info",
|
||||
"aretrieve_batch",
|
||||
"aretrieve_fine_tuning_job",
|
||||
"arm_routing_read_prefetch",
|
||||
"arun_thread",
|
||||
"async_get_available_deployment",
|
||||
"async_get_fully_unhealthy_model_names",
|
||||
"async_get_healthy_deployments",
|
||||
"async_pre_routing_hook",
|
||||
"auto_routers",
|
||||
"avector_store_create",
|
||||
"avector_store_search",
|
||||
"background_health_check_model_groups",
|
||||
"bind_to_request_logging",
|
||||
"cache",
|
||||
"cache_responses",
|
||||
"cached_deployment_model_info",
|
||||
"cached_model_group_info",
|
||||
"completion",
|
||||
"complexity_routers",
|
||||
"config_deployments",
|
||||
"content_policy_fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"cooldown_time",
|
||||
"default_deployment",
|
||||
"default_litellm_params",
|
||||
"delete_deployment",
|
||||
"deployment_latency_map",
|
||||
"deployment_names",
|
||||
"deployment_usable_by_team",
|
||||
"deployments_for_request",
|
||||
"deployments_named",
|
||||
"discard",
|
||||
"embedding",
|
||||
"enable_health_check_routing",
|
||||
"fallbacks",
|
||||
"generate_model_id",
|
||||
"get_all_deployments",
|
||||
"get_available_deployment_for_pass_through",
|
||||
"get_available_guardrail",
|
||||
"get_candidate_model_ids_for_route",
|
||||
"get_configured_display_name",
|
||||
"get_configured_mode",
|
||||
"get_configured_service_tiers",
|
||||
"get_configured_token_limits",
|
||||
"get_credential_deployment",
|
||||
"get_deployment",
|
||||
"get_deployment_by_model_group_name",
|
||||
"get_deployment_credentials",
|
||||
"get_deployment_credentials_with_provider",
|
||||
"get_deployment_model_info",
|
||||
"get_discovered_model_info",
|
||||
"get_fully_blocked_model_names",
|
||||
"get_model_access_groups",
|
||||
"get_model_from_alias",
|
||||
"get_model_group_info",
|
||||
"get_model_group_usage",
|
||||
"get_model_ids",
|
||||
"get_model_info",
|
||||
"get_model_list",
|
||||
"get_model_list_from_model_alias",
|
||||
"get_model_list_from_routing_groups",
|
||||
"get_model_list_of_routed_group",
|
||||
"get_model_listing_info",
|
||||
"get_model_names",
|
||||
"get_remaining_model_group_usage",
|
||||
"get_routable_deployments",
|
||||
"get_routable_upstream_model",
|
||||
"get_router_deployment_budget_limiter",
|
||||
"get_routing_group",
|
||||
"get_settings",
|
||||
"get_valid_args",
|
||||
"guardrail_list",
|
||||
"has_model_id",
|
||||
"health_check_ignore_transient_errors",
|
||||
"health_state_cache",
|
||||
"is_deployment_blocked",
|
||||
"is_model_access_group_for_wildcard_route",
|
||||
"is_recognized_model",
|
||||
"kubernetes_pod_discovery",
|
||||
"map_team_model",
|
||||
"model_group_alias",
|
||||
"model_list",
|
||||
"model_name_to_deployment_indices",
|
||||
"model_names",
|
||||
"pattern_router",
|
||||
"priority_header_update",
|
||||
"prompt_caching_pin_cache",
|
||||
"provider_budget_config",
|
||||
"quality_routers",
|
||||
"resolve_model_name_from_model_id",
|
||||
"resolve_served_model_name",
|
||||
"resolved_litellm_models",
|
||||
"routable_model_group",
|
||||
"router_general_settings",
|
||||
"routing_groups",
|
||||
"routing_plugins",
|
||||
"schedule_acompletion",
|
||||
"search_tools",
|
||||
"serve_through_request_logging",
|
||||
"should_include_deployment",
|
||||
"split_request_priority",
|
||||
"team_model_to_deployment_indices",
|
||||
"team_pattern_routers",
|
||||
"team_ptu_ceiling",
|
||||
"team_public_model_names",
|
||||
"team_public_model_names_of",
|
||||
"update_redis_cache",
|
||||
"update_settings",
|
||||
"upsert_deployment",
|
||||
"wildcard_route_deployments",
|
||||
"wildcard_routed_models"
|
||||
],
|
||||
"members_router_family": [
|
||||
"_count_pre_call_check_tokens",
|
||||
"_pre_call_checks",
|
||||
"_update_kwargs_before_fallbacks",
|
||||
"acompletion",
|
||||
"adecisions",
|
||||
"aembedding",
|
||||
"allowed_fails",
|
||||
"allowed_fails_policy",
|
||||
"aresponses",
|
||||
"async_function_with_fallbacks",
|
||||
"async_get_healthy_deployments",
|
||||
"cache",
|
||||
"cached_deployment_model_info",
|
||||
"cooldown_cache",
|
||||
"cooldown_time",
|
||||
"default_max_parallel_requests",
|
||||
"deployments_named",
|
||||
"disable_cooldowns",
|
||||
"embedding",
|
||||
"enable_pre_call_checks",
|
||||
"enable_tag_filtering",
|
||||
"explicit_allowed_fails",
|
||||
"fallback_access_check",
|
||||
"fallback_budget_check",
|
||||
"fallbacks",
|
||||
"get_all_deployments",
|
||||
"get_allowed_fails_from_policy",
|
||||
"get_deployment",
|
||||
"get_model_group",
|
||||
"get_model_ids",
|
||||
"get_model_info",
|
||||
"get_model_list",
|
||||
"get_router_model_info",
|
||||
"log_retry",
|
||||
"model_list",
|
||||
"provider_default_deployment_ids",
|
||||
"routing_group_has_alternatives",
|
||||
"routing_strategy",
|
||||
"search_tools",
|
||||
"set_custom_routing_strategy",
|
||||
"slack_alerting_logger",
|
||||
"tag_filtering_match_any",
|
||||
"tag_routing_prefix",
|
||||
"team_model_has_alternatives"
|
||||
],
|
||||
"router_utils_imports_outside": [
|
||||
"litellm.router_utils.access_windows",
|
||||
"litellm.router_utils.add_retry_fallback_headers",
|
||||
"litellm.router_utils.clientside_credential_handler",
|
||||
"litellm.router_utils.common_utils",
|
||||
"litellm.router_utils.cooldown_handlers",
|
||||
"litellm.router_utils.deployment_pricing",
|
||||
"litellm.router_utils.fallback_event_handlers",
|
||||
"litellm.router_utils.get_retry_from_policy",
|
||||
"litellm.router_utils.health_state_cache",
|
||||
"litellm.router_utils.pattern_match_deployments",
|
||||
"litellm.router_utils.ptu_shares",
|
||||
"litellm.router_utils.reasoning_effort_capability",
|
||||
"litellm.router_utils.router_callbacks.track_deployment_metrics",
|
||||
"litellm.router_utils.routing_groups",
|
||||
"litellm.router_utils.search_api_router"
|
||||
]
|
||||
}
|
||||
235
tests/code_coverage_tests/router_surface_check.py
Normal file
235
tests/code_coverage_tests/router_surface_check.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
"""
|
||||
Fence for the Router's public surface.
|
||||
|
||||
Code outside the router may use only the Router members listed in
|
||||
router_surface_allowlist.json, and may import only the litellm.router_utils
|
||||
modules listed there. The lists hold what is used today and may only shrink:
|
||||
|
||||
- a member or module used outside the router that is not listed fails the
|
||||
check (use an existing entry point instead, or discuss widening the surface);
|
||||
- a listed member or module that nothing outside the router uses any more
|
||||
also fails, so the entry is removed in the same change.
|
||||
|
||||
"Outside" means every non-test file under litellm/ and enterprise/ except
|
||||
litellm/router.py and the router's own modules. Uses from the router family
|
||||
(litellm/router_utils, litellm/router_strategy) are listed separately, because
|
||||
those modules become private parts of the router core in a later step.
|
||||
|
||||
Member uses are found the same way the boundary survey found them: attribute
|
||||
access on a receiver whose name looks like a router (llm_router,
|
||||
litellm_router_instance, router, ...) or on the Router class, plus
|
||||
getattr/setattr/hasattr with a constant name on such a receiver. Members
|
||||
include the container file endpoints Router.__init__ registers by name.
|
||||
|
||||
Run: python tests/code_coverage_tests/router_surface_check.py [--write]
|
||||
--write regenerates the allowlist from the current tree (for shrinking it).
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, List, Set
|
||||
|
||||
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.json")
|
||||
|
||||
ROUTER_FILES = {
|
||||
"litellm/router.py",
|
||||
}
|
||||
ROUTER_CLASS_FILES = ["litellm/router.py"]
|
||||
ROUTER_CLASS_NAMES = {"Router"}
|
||||
|
||||
SCAN_ROOTS = ["litellm", "enterprise"]
|
||||
SKIP_DIRS = {
|
||||
".git",
|
||||
"node_modules",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"litellm-rust",
|
||||
"ui",
|
||||
"docs",
|
||||
"cookbook",
|
||||
"tests",
|
||||
"test",
|
||||
}
|
||||
|
||||
RECEIVER = re.compile(
|
||||
r"(^|[._])(llm_router|router|litellm_router|litellm_router_instance|router_instance"
|
||||
r"|_router|proxy_router|llm_router_instance|router_obj|mock_router|Router)$"
|
||||
)
|
||||
# Router.__init__ registers these with setattr(self, name, ...), so they never appear as self.<name>.
|
||||
CONTAINER_FILE_ENDPOINTS = "litellm/containers/endpoints.json"
|
||||
DYNAMIC_FUNCS = {"getattr", "setattr", "hasattr"}
|
||||
|
||||
|
||||
def _area(rel: str) -> str:
|
||||
if rel.startswith(("litellm/router_utils/", "litellm/router_strategy/")):
|
||||
return "router_family"
|
||||
return "outside"
|
||||
|
||||
|
||||
def _is_test(rel: str) -> bool:
|
||||
name = os.path.basename(rel)
|
||||
return "/tests/" in rel or name.startswith("test_") or name.endswith("_test.py")
|
||||
|
||||
|
||||
def router_members() -> Set[str]:
|
||||
members: Set[str] = set()
|
||||
for rel in ROUTER_CLASS_FILES:
|
||||
with open(os.path.join(ROOT, rel), encoding="utf-8") as f:
|
||||
tree = ast.parse(f.read())
|
||||
for cls in tree.body:
|
||||
if not isinstance(cls, ast.ClassDef) or cls.name not in ROUTER_CLASS_NAMES:
|
||||
continue
|
||||
for node in cls.body:
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
members.add(node.name)
|
||||
elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
||||
members.add(node.target.id)
|
||||
elif isinstance(node, ast.Assign):
|
||||
members.update(t.id for t in node.targets if isinstance(t, ast.Name))
|
||||
for node in ast.walk(cls):
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
and isinstance(node.ctx, ast.Store)
|
||||
):
|
||||
members.add(node.attr)
|
||||
with open(os.path.join(ROOT, CONTAINER_FILE_ENDPOINTS), encoding="utf-8") as f:
|
||||
for endpoint in json.load(f)["endpoints"]:
|
||||
members.update((endpoint["name"], endpoint["async_name"]))
|
||||
return {m for m in members if not (m.startswith("__") and m.endswith("__"))}
|
||||
|
||||
|
||||
def _python_files():
|
||||
for top in SCAN_ROOTS:
|
||||
base = os.path.join(ROOT, top)
|
||||
for dirpath, dirnames, filenames in os.walk(base):
|
||||
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
||||
for fn in filenames:
|
||||
if fn.endswith(".py"):
|
||||
path = os.path.join(dirpath, fn)
|
||||
yield os.path.relpath(path, ROOT).replace(os.sep, "/"), path
|
||||
|
||||
|
||||
def scan() -> Dict[str, Dict[str, Set[str]]]:
|
||||
"""Return {"members": {area: {member: files}}, "router_utils_imports": {module: files}}."""
|
||||
members = router_members()
|
||||
uses: Dict[str, Dict[str, Set[str]]] = {"outside": {}, "router_family": {}}
|
||||
imports: Dict[str, Set[str]] = {}
|
||||
for rel, path in _python_files():
|
||||
if rel in ROUTER_FILES or _is_test(rel):
|
||||
continue
|
||||
with open(path, encoding="utf-8") as f:
|
||||
src = f.read()
|
||||
has_router = "router" in src.lower()
|
||||
if not has_router:
|
||||
continue
|
||||
try:
|
||||
tree = ast.parse(src)
|
||||
except SyntaxError:
|
||||
continue
|
||||
area = _area(rel)
|
||||
for name in member_uses(tree, members):
|
||||
uses[area].setdefault(name, set()).add(rel)
|
||||
if area == "outside":
|
||||
for node in ast.walk(tree):
|
||||
for mod in _router_utils_modules(node):
|
||||
imports.setdefault(mod, set()).add(rel)
|
||||
return {"members": uses, "router_utils_imports": imports}
|
||||
|
||||
|
||||
def member_uses(tree: ast.AST, members: Set[str]) -> Set[str]:
|
||||
"""The Router members a module uses, on an instance or on the Router class itself."""
|
||||
used: Set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Attribute) and node.attr in members:
|
||||
if RECEIVER.search(ast.unparse(node.value)):
|
||||
used.add(node.attr)
|
||||
elif (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id in DYNAMIC_FUNCS
|
||||
and len(node.args) >= 2
|
||||
and isinstance(node.args[1], ast.Constant)
|
||||
and node.args[1].value in members
|
||||
and RECEIVER.search(ast.unparse(node.args[0]))
|
||||
):
|
||||
used.add(node.args[1].value)
|
||||
return used
|
||||
|
||||
|
||||
def _router_utils_modules(node: ast.AST) -> List[str]:
|
||||
if isinstance(node, ast.ImportFrom) and node.module:
|
||||
if node.module == "litellm.router_utils":
|
||||
return ["litellm.router_utils." + a.name for a in node.names]
|
||||
if node.module.startswith("litellm.router_utils."):
|
||||
return [node.module]
|
||||
if isinstance(node, ast.Import):
|
||||
return [a.name for a in node.names if a.name.startswith("litellm.router_utils.")]
|
||||
return []
|
||||
|
||||
|
||||
def build_allowlist(found) -> dict:
|
||||
return {
|
||||
"_doc": (
|
||||
"Router members and litellm.router_utils modules used outside the router. "
|
||||
"Shrink-only: see tests/code_coverage_tests/router_surface_check.py."
|
||||
),
|
||||
"members_outside": sorted(found["members"]["outside"]),
|
||||
"members_router_family": sorted(found["members"]["router_family"]),
|
||||
"router_utils_imports_outside": sorted(found["router_utils_imports"]),
|
||||
}
|
||||
|
||||
|
||||
def compare(allow: dict, found) -> List[str]:
|
||||
errors: List[str] = []
|
||||
sections = [
|
||||
("members_outside", found["members"]["outside"], "Router member"),
|
||||
("members_router_family", found["members"]["router_family"], "Router member (router family)"),
|
||||
("router_utils_imports_outside", found["router_utils_imports"], "litellm.router_utils import"),
|
||||
]
|
||||
for key, used, what in sections:
|
||||
listed = set(allow.get(key, []))
|
||||
for name in sorted(set(used) - listed):
|
||||
files = ", ".join(sorted(used[name]))
|
||||
errors.append(
|
||||
f"new {what} outside the router: {name!r} in {files}. The router surface may only "
|
||||
f"shrink: use an entry point instead."
|
||||
)
|
||||
for name in sorted(listed - set(used)):
|
||||
errors.append(
|
||||
f"{what} {name!r} is listed in {os.path.basename(ALLOWLIST)} [{key}] but nothing uses it "
|
||||
f"any more: remove it from the list."
|
||||
)
|
||||
return errors
|
||||
|
||||
|
||||
def main(argv: List[str]) -> int:
|
||||
found = scan()
|
||||
if "--write" in argv:
|
||||
with open(ALLOWLIST, "w", encoding="utf-8") as f:
|
||||
json.dump(build_allowlist(found), f, indent=2)
|
||||
f.write("\n")
|
||||
print(f"wrote {ALLOWLIST}")
|
||||
return 0
|
||||
with open(ALLOWLIST, encoding="utf-8") as f:
|
||||
allow = json.load(f)
|
||||
errors = compare(allow, found)
|
||||
for e in errors:
|
||||
print(e)
|
||||
if errors:
|
||||
return 1
|
||||
print(
|
||||
f"router surface ok: {len(allow['members_outside'])} members outside, "
|
||||
f"{len(allow['members_router_family'])} from the router family, "
|
||||
f"{len(allow['router_utils_imports_outside'])} router_utils modules imported outside"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
|
@ -129,4 +129,4 @@ litellm/proxy/utils.py prefetch_config_params prisma param_name.in `param_names`
|
|||
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma [scope.entity_id_field].in `list(scope.entity_ids)` 0
|
||||
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma api_key.in `list(scope.api_keys)` 0
|
||||
litellm/repositories/daily_activity_repository.py DailyActivityRepository.daily_rows prisma not.in `list(scope.exclude_entity_ids)` 0
|
||||
litellm/router_utils/auto_router_model_naming.py <module> raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0
|
||||
litellm/router_strategy/auto_router/auto_router_model_naming.py <module> raw-sql classifier_type.IN `IN ({_LLM_CLASSIFIER_TYPES_SQL})` 0
|
||||
|
|
|
|||
|
|
@ -26,6 +26,20 @@ _DECISIONS_PROBE_REPLY: Final = JsonResponse(
|
|||
"usage": {"input_tokens": 10, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
_EXTRACTION_PROBE_REPLY: Final = JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"result": {
|
||||
"model": "clairic",
|
||||
"answers": {"reachable": {"type": "text", "value": "health", "confidence": 0.9}},
|
||||
"truncated": False,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 0},
|
||||
},
|
||||
"success": True,
|
||||
"errors": [],
|
||||
"messages": [],
|
||||
},
|
||||
)
|
||||
_CONFIGURED_PROBE_REPLY: Final = JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
|
|
@ -287,6 +301,29 @@ def test_decisions_mode_health_check_resolves_the_mode_from_the_cost_map_and_sen
|
|||
]
|
||||
|
||||
|
||||
def test_extraction_mode_health_check_resolves_the_mode_from_the_cost_map_and_sends_the_extraction_probe(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
handle: Final = register_scenario(f"health-extraction-{uuid.uuid4().hex[:12]}", _EXTRACTION_PROBE_REPLY)
|
||||
scenario.cleanups.callback(delete_scenario, handle)
|
||||
model: Final = scenario.model(
|
||||
model="cloudflare/@cf/cloudflare/clairic", api_base=handle.api_base(), api_key="synthetic-cloudflare-key"
|
||||
)
|
||||
report: Final = _health_report(gateway, model)
|
||||
assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
|
||||
assert _probes_sent_to(gateway, handle) == [
|
||||
(
|
||||
f"/{handle.scenario_id}/ai/run/@cf/cloudflare/clairic",
|
||||
{
|
||||
"model": "clairic",
|
||||
"state": os.environ.get("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"),
|
||||
"fields": {"reachable": {"type": "text", "instructions": "Any word from the text"}},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation"))
|
||||
def test_decisions_mode_health_check_sends_the_configured_state_and_questions(gateway: Gateway, mode: str) -> None:
|
||||
with gateway.scenario() as scenario:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
|
|||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from hashlib import sha256
|
||||
from itertools import chain
|
||||
from itertools import chain, count
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal, TypeVar
|
||||
|
||||
|
|
@ -50,6 +50,7 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat
|
|||
directory: Final = tmp_path_factory.mktemp("mcp-llm-gateway")
|
||||
stock: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
config: Final = directory / "stored-responses.yaml"
|
||||
(directory / f"{STREAM_BLOCK_MODULE}.py").write_text(STREAM_BLOCK_SOURCE)
|
||||
config.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
|
|
@ -57,8 +58,17 @@ def _llm_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gat
|
|||
"general_settings": {**object_value(stock["general_settings"]), "store_prompts_in_spend_logs": True},
|
||||
"litellm_settings": {
|
||||
**object_value(stock["litellm_settings"]),
|
||||
"cost_margin_config": {"hosted_vllm": {"fixed_amount": FIXED_MARGIN}},
|
||||
"cost_margin_config": {
|
||||
"hosted_vllm": {"fixed_amount": FIXED_MARGIN},
|
||||
"deepseek": {"fixed_amount": FIXED_MARGIN},
|
||||
},
|
||||
},
|
||||
"guardrails": [
|
||||
{
|
||||
"guardrail_name": STREAM_BLOCK,
|
||||
"litellm_params": {"guardrail": f"{STREAM_BLOCK_MODULE}.BlockGuard", "mode": "post_call"},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
@ -79,7 +89,7 @@ def gateway(_llm_gateway: tuple[Gateway, Mapping[str, str]]) -> Iterator[Gateway
|
|||
yield private
|
||||
|
||||
|
||||
Surface = Literal["chat", "responses", "messages", "messages_bridge"]
|
||||
Surface = Literal["chat", "responses", "messages", "messages_bridge", "responses_bridge"]
|
||||
SURFACES: Final[tuple[Surface, ...]] = ("chat", "responses", "messages", "messages_bridge")
|
||||
ADD: Final = {"a": 2, "b": 3}
|
||||
ANSWER: Final = "the sum is 5"
|
||||
|
|
@ -88,6 +98,19 @@ AUTO: Final = {**GATEWAY_REF, "require_approval": "never"}
|
|||
OUTAGE: Final = "bridge-outage"
|
||||
DIRECT: Final = "answer-without-tools"
|
||||
FIXED_MARGIN: Final = 0.25
|
||||
STREAM_BLOCK: Final = "mcp-stream-block"
|
||||
STREAM_BLOCK_MODULE: Final = "mcp_stream_block"
|
||||
STREAM_BLOCK_DETAIL: Final = "blocked by output policy"
|
||||
STREAM_BLOCK_SOURCE: Final = (
|
||||
"from fastapi import HTTPException\n\n"
|
||||
"from litellm.integrations.custom_guardrail import CustomGuardrail\n\n\n"
|
||||
"class BlockGuard(CustomGuardrail):\n"
|
||||
" async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):\n"
|
||||
" async for _ in response:\n"
|
||||
" pass\n"
|
||||
f' raise HTTPException(status_code=400, detail="{STREAM_BLOCK_DETAIL}")\n'
|
||||
" yield\n"
|
||||
)
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
|
@ -128,6 +151,8 @@ class Billing:
|
|||
first: tuple[int, int] = (10, 5)
|
||||
follow_up: tuple[int, int] = (10, 5)
|
||||
follow_up_fails: bool = False
|
||||
follow_up_breaks: bool = False
|
||||
tool_rounds: int = 1
|
||||
|
||||
|
||||
EVEN_ROUNDS: Final = Billing()
|
||||
|
|
@ -138,6 +163,10 @@ def _fixed_turn(tool: str) -> Callable[[Mapping[str, JsonValue]], Turn]:
|
|||
return lambda _: Turn(tool, json.dumps(ADD), ANSWER)
|
||||
|
||||
|
||||
def _slow_turn(tool: str, seconds: float) -> Callable[[Mapping[str, JsonValue]], Turn]:
|
||||
return lambda _: Turn(tool, json.dumps({"seconds": seconds}), ANSWER)
|
||||
|
||||
|
||||
def _echoing_turn(body: Mapping[str, JsonValue]) -> Turn:
|
||||
names: Final = _tool_names(body)
|
||||
return Turn(names[0] if names else "", json.dumps({"query": _prompt(body)}), _tool_result_text(body) or "")
|
||||
|
|
@ -183,6 +212,50 @@ def _item_events(index: int, item: Mapping[str, JsonValue]) -> tuple[dict[str, J
|
|||
)
|
||||
|
||||
|
||||
def _event_stream(events: Sequence[Mapping[str, JsonValue]]) -> Reply:
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(
|
||||
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()
|
||||
for number, event in enumerate(events)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _broken_follow_up(target: str, identity: str, tokens: tuple[int, int]) -> Reply:
|
||||
outage: Final = {"message": FOLLOW_UP_OUTAGE, "type": "server_error", "code": None}
|
||||
if target.endswith("/chat/completions"):
|
||||
delta: Final = {"role": "assistant", "content": "the sum"}
|
||||
started: Final = {
|
||||
"id": f"chatcmpl-{identity}",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "delta": delta, "finish_reason": None}],
|
||||
}
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(f"data: {json.dumps(frame)}\n\n".encode() for frame in (started, {"error": outage})),
|
||||
)
|
||||
input_tokens, output_tokens = tokens
|
||||
failed: Final[dict[str, JsonValue]] = {
|
||||
"id": f"resp_{identity}",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": "failed",
|
||||
"model": "gpt-4o-mini",
|
||||
"output": [],
|
||||
"error": {"code": "server_error", "message": FOLLOW_UP_OUTAGE},
|
||||
"usage": {"input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": sum(tokens)},
|
||||
}
|
||||
return _event_stream(
|
||||
(
|
||||
{"type": "response.created", "response": {**failed, "status": "in_progress", "error": None, "usage": None}},
|
||||
{"type": "response.failed", "response": failed},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _responses_stream(response: Mapping[str, JsonValue], items: Sequence[Mapping[str, JsonValue]]) -> Reply:
|
||||
item_events: Final = tuple(chain.from_iterable(_item_events(index, item) for index, item in enumerate(items)))
|
||||
events: Final = (
|
||||
|
|
@ -191,13 +264,7 @@ def _responses_stream(response: Mapping[str, JsonValue], items: Sequence[Mapping
|
|||
*item_events,
|
||||
{"type": "response.completed", "response": response},
|
||||
)
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=tuple(
|
||||
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()
|
||||
for number, event in enumerate(events)
|
||||
),
|
||||
)
|
||||
return _event_stream(events)
|
||||
|
||||
|
||||
def _block_ids(message: JsonValue, kind: str, field: str) -> frozenset[str]:
|
||||
|
|
@ -252,6 +319,8 @@ def _anthropic_tool_result_rule(body: Mapping[str, JsonValue]) -> Reply | None:
|
|||
def _model_double(
|
||||
plan: Callable[[Mapping[str, JsonValue]], Turn], billing: Billing = EVEN_ROUNDS, parallel: int = 1
|
||||
) -> Callable[[Request], Reply]:
|
||||
served: Final = count()
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target.endswith("/models"):
|
||||
return _json({"object": "list", "data": []})
|
||||
|
|
@ -260,11 +329,12 @@ def _model_double(
|
|||
outage: Final = {"error": {"message": "scripted provider outage", "type": "server_error", "code": None}}
|
||||
return Reply(status=500, body=json.dumps(outage).encode())
|
||||
turn: Final = plan(body)
|
||||
call_number: Final = next(served)
|
||||
answered: Final = _has_tool_result(body)
|
||||
if answered and billing.follow_up_fails:
|
||||
failure: Final = {"error": {"message": FOLLOW_UP_OUTAGE, "type": "server_error", "code": None}}
|
||||
return Reply(status=500, body=json.dumps(failure).encode())
|
||||
done: Final = answered or DIRECT in _prompt(body)
|
||||
done: Final = DIRECT in _prompt(body) or (answered and call_number >= billing.tool_rounds)
|
||||
calls: Final = range(1, parallel + 1)
|
||||
prompt_tokens, completion_tokens = billing.follow_up if answered else billing.first
|
||||
identity: Final = uuid.uuid4().hex[:12]
|
||||
|
|
@ -273,9 +343,15 @@ def _model_double(
|
|||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": prompt_tokens + completion_tokens,
|
||||
}
|
||||
if answered and billing.follow_up_breaks:
|
||||
return _broken_follow_up(request.target, identity, (prompt_tokens, completion_tokens))
|
||||
if request.target.endswith("/chat/completions"):
|
||||
requested: Final = tuple(
|
||||
{"id": f"call_{call}", "type": "function", "function": {"name": turn.tool, "arguments": turn.arguments}}
|
||||
{
|
||||
"id": f"call_{identity}_{call}",
|
||||
"type": "function",
|
||||
"function": {"name": turn.tool, "arguments": turn.arguments},
|
||||
}
|
||||
for call in calls
|
||||
)
|
||||
message: Final = (
|
||||
|
|
@ -460,6 +536,7 @@ def _upstream_model(surface: Surface) -> str:
|
|||
"responses": "openai/responses/gpt-4o-mini",
|
||||
"messages": "anthropic/claude-sonnet-4-5",
|
||||
"messages_bridge": "hosted_vllm/gpt-4o-mini",
|
||||
"responses_bridge": "deepseek/gpt-4o-mini",
|
||||
}[surface]
|
||||
|
||||
|
||||
|
|
@ -475,11 +552,15 @@ def _rig(
|
|||
billing: Billing = EVEN_ROUNDS,
|
||||
prices: Mapping[str, JsonValue] | None = None,
|
||||
parallel: int = 1,
|
||||
slow_tool_seconds: float | None = None,
|
||||
) -> Iterator[Rig]:
|
||||
alias: Final = "llm" + uuid.uuid4().hex[:8]
|
||||
plan: Final = (
|
||||
_fixed_turn(f"{alias}-add") if slow_tool_seconds is None else _slow_turn(f"{alias}-slow", slow_tool_seconds)
|
||||
)
|
||||
with (
|
||||
mcp_peer() as peer,
|
||||
wire_server(_model_double(_fixed_turn(f"{alias}-add"), billing, parallel)) as wire,
|
||||
mcp_peer(rich=slow_tool_seconds is not None) as peer,
|
||||
wire_server(_model_double(plan, billing, parallel)) as wire,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
server_id: Final = register_mcp(scenario, peer, alias)
|
||||
|
|
@ -1048,6 +1129,262 @@ def test_bridged_next_turn_replays_the_mcp_turn_answer(gateway: Gateway) -> None
|
|||
assert ANSWER in _assistant_texts(opening), opening
|
||||
|
||||
|
||||
StreamedUpstream = Literal["messages_bridge", "responses_bridge"]
|
||||
STREAMED_UPSTREAMS: Final[tuple[StreamedUpstream, ...]] = ("messages_bridge", "responses_bridge")
|
||||
STREAMED_IDS: Final = ("native", "bridged")
|
||||
SLOW_TOOL_SECONDS: Final = 5.0
|
||||
|
||||
|
||||
def _events(response: httpx.Response) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
JSON_OBJECT.validate_json(line.removeprefix("data: "))
|
||||
for line in response.text.splitlines()
|
||||
if line.startswith("data: ") and line != "data: [DONE]"
|
||||
)
|
||||
|
||||
|
||||
def _completed(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
finished: Final = tuple(
|
||||
object_value(event["response"]) for event in _events(response) if event["type"] == "response.completed"
|
||||
)
|
||||
assert len(finished) == 1, response.text
|
||||
return finished[0]
|
||||
|
||||
|
||||
def _model_rows(call_id: str, settled: Callable[[list[dict[str, JsonValue]]], bool]) -> list[dict[str, JsonValue]]:
|
||||
return eventually(
|
||||
lambda: read_rows(
|
||||
"SELECT request_id, status, spend, prompt_tokens, completion_tokens "
|
||||
'FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s AND call_type NOT IN (%s, %s)',
|
||||
(call_id, "call_mcp_tool", "list_mcp_tools"),
|
||||
),
|
||||
settled,
|
||||
seconds=70,
|
||||
return_last_on_timeout=True,
|
||||
)
|
||||
|
||||
|
||||
def _billed(rows: Sequence[Mapping[str, JsonValue]]) -> float:
|
||||
return sum(float(str(row["spend"])) for row in rows)
|
||||
|
||||
|
||||
def _bills_at_least(expected: float) -> Callable[[list[dict[str, JsonValue]]], bool]:
|
||||
return lambda rows: bool(rows) and _billed(rows) >= expected - 1e-9
|
||||
|
||||
|
||||
def _statuses(rows: Sequence[Mapping[str, JsonValue]]) -> list[str]:
|
||||
return sorted(str(row["status"]) for row in rows)
|
||||
|
||||
|
||||
def _logs_a_failure_billing(expected: float) -> Callable[[list[dict[str, JsonValue]]], bool]:
|
||||
return lambda rows: "failure" in _statuses(rows) and _billed(rows) >= expected - 1e-9
|
||||
|
||||
|
||||
def _limited_key(rig: Rig) -> str:
|
||||
return rig.scenario.key(object_permission={"mcp_servers": [rig.server_id]}, max_parallel_requests=1)
|
||||
|
||||
|
||||
def _answers_directly(rig: Rig, key: str) -> int:
|
||||
return rig.send(key, [AUTO], prompt=f"{DIRECT} {rig.alias}").status_code
|
||||
|
||||
|
||||
STREAMED_ROUNDS: Final = (
|
||||
pytest.param("messages_bridge", 1, id="native"),
|
||||
pytest.param("responses_bridge", 1, id="bridged"),
|
||||
pytest.param("messages_bridge", 2, id="native-two-tool-rounds"),
|
||||
pytest.param("responses_bridge", 2, id="bridged-two-tool-rounds"),
|
||||
pytest.param("messages_bridge", 0, id="native-no-tool-call"),
|
||||
pytest.param("responses_bridge", 0, id="bridged-no-tool-call"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("upstream", "tool_rounds"), STREAMED_ROUNDS)
|
||||
def test_streaming_responses_auto_execute_bills_every_round_once_on_one_success_row(
|
||||
gateway: Gateway, upstream: StreamedUpstream, tool_rounds: int
|
||||
) -> None:
|
||||
rounds: Final = (FIRST_ROUND, *(FOLLOW_UP_ROUND,) * tool_rounds)
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, tool_rounds=max(tool_rounds, 1))
|
||||
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
|
||||
prompt: Final = None if tool_rounds else f"{DIRECT} {rig.alias}"
|
||||
response: Final = rig.send(_granted_key(rig), [AUTO], prompt=prompt, stream=True)
|
||||
assert response.status_code == 200, response.text
|
||||
completed: Final = _completed(response)
|
||||
assert rig.final_text(completed) == ANSWER, response.text
|
||||
assert len(rig.upstream_tools()) == len(rounds)
|
||||
assert len(_peer_add_calls(rig.peer)) == tool_rounds
|
||||
expected: Final = _price(*rounds) + FIXED_MARGIN
|
||||
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _bills_at_least(expected))
|
||||
assert [row["status"] for row in rows] == ["success"], rows
|
||||
row: Final = rows[0]
|
||||
assert (row["prompt_tokens"], row["completion_tokens"]) == (
|
||||
sum(prompt_tokens for prompt_tokens, _ in rounds),
|
||||
sum(completion_tokens for _, completion_tokens in rounds),
|
||||
), row
|
||||
usage: Final = object_value(completed["usage"])
|
||||
assert (usage["input_tokens"], usage["output_tokens"]) == (row["prompt_tokens"], row["completion_tokens"])
|
||||
assert float(str(row["spend"])) == pytest.approx(expected), row
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
|
||||
def test_streaming_responses_rejected_follow_up_bills_round_one_once(
|
||||
gateway: Gateway, upstream: StreamedUpstream
|
||||
) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_fails=True)
|
||||
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
|
||||
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True)
|
||||
assert FOLLOW_UP_OUTAGE in response.text, response.text
|
||||
assert ANSWER not in response.text, response.text
|
||||
assert len(rig.upstream_tools()) == 2
|
||||
expected: Final = _price(FIRST_ROUND) + FIXED_MARGIN
|
||||
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _bills_at_least(expected))
|
||||
assert _billed(rows) == pytest.approx(expected), rows
|
||||
|
||||
|
||||
BROKEN_FOLLOW_UPS: Final = (
|
||||
pytest.param("messages_bridge", FOLLOW_UP_ROUND, id="native"),
|
||||
pytest.param("responses_bridge", None, id="bridged"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("upstream", "reported_usage"), BROKEN_FOLLOW_UPS)
|
||||
def test_streaming_responses_follow_up_broken_mid_stream_bills_round_one_once_and_itself_on_a_failure_row(
|
||||
gateway: Gateway, upstream: StreamedUpstream, reported_usage: tuple[int, int] | None
|
||||
) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND, follow_up_breaks=True)
|
||||
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
|
||||
response: Final = rig.send(_granted_key(rig), [AUTO], stream=True)
|
||||
assert ANSWER not in response.text, response.text
|
||||
assert len(rig.upstream_tools()) == 2
|
||||
round_one: Final = _price(FIRST_ROUND) + FIXED_MARGIN
|
||||
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(round_one))
|
||||
assert _statuses(rows) == ["failure", "success"], rows
|
||||
(success,) = (row for row in rows if row["status"] == "success")
|
||||
(failure,) = (row for row in rows if row["status"] == "failure")
|
||||
assert (success["prompt_tokens"], success["completion_tokens"]) == FIRST_ROUND, rows
|
||||
assert float(str(success["spend"])) == pytest.approx(round_one), rows
|
||||
failed_round: Final = (int(str(failure["prompt_tokens"])), int(str(failure["completion_tokens"])))
|
||||
assert min(failed_round) > 0, rows
|
||||
assert reported_usage in (None, failed_round), rows
|
||||
assert float(str(failure["spend"])) == pytest.approx(_price(failed_round)), rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
|
||||
def test_streaming_responses_blocked_by_a_post_call_guardrail_bills_every_round_once(
|
||||
gateway: Gateway, upstream: StreamedUpstream
|
||||
) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
|
||||
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
|
||||
key: Final = _granted_key(rig)
|
||||
one_round: Final = rig.send(key, [], prompt=f"{DIRECT} {rig.alias}", stream=True, guardrails=[STREAM_BLOCK])
|
||||
assert STREAM_BLOCK_DETAIL in one_round.text, one_round.text
|
||||
one_round_rows: Final = _model_rows(
|
||||
one_round.headers["x-litellm-call-id"], _logs_a_failure_billing(_price(FIRST_ROUND) + FIXED_MARGIN)
|
||||
)
|
||||
rig.wire.drain()
|
||||
response: Final = rig.send(key, [AUTO], stream=True, guardrails=[STREAM_BLOCK])
|
||||
assert STREAM_BLOCK_DETAIL in response.text, response.text
|
||||
assert len(rig.upstream_tools()) == 2 and len(_peer_add_calls(rig.peer)) == 1
|
||||
expected: Final = _price(FIRST_ROUND, FOLLOW_UP_ROUND) + FIXED_MARGIN
|
||||
rows: Final = _model_rows(response.headers["x-litellm-call-id"], _logs_a_failure_billing(expected))
|
||||
assert _statuses(rows) == _statuses(one_round_rows), (rows, one_round_rows)
|
||||
assert _billed(rows) == pytest.approx(expected), rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("surface", "stream"),
|
||||
(("responses", True), ("responses", False), ("chat", False)),
|
||||
ids=("responses-stream", "responses", "chat"),
|
||||
)
|
||||
def test_auto_executed_tool_call_runs_inside_the_requests_one_parallel_slot(
|
||||
gateway: Gateway, surface: Surface, stream: bool
|
||||
) -> None:
|
||||
with _rig(gateway, surface, slow_tool_seconds=SLOW_TOOL_SECONDS) as rig:
|
||||
key: Final = _limited_key(rig)
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
running: Final = pool.submit(rig.send, key, [AUTO], None, stream=stream)
|
||||
eventually(lambda: _peer_add_calls(rig.peer), lambda found: len(found) == 1, seconds=30)
|
||||
during: Final = _answers_directly(rig, key)
|
||||
served: Final = running.result(timeout=90)
|
||||
assert during == 429
|
||||
assert served.status_code == 200 and ANSWER in served.text, served.text
|
||||
eventually(lambda: _answers_directly(rig, key), lambda status: status == 200, seconds=30)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
|
||||
def test_streaming_responses_hung_up_during_the_tool_call_bills_each_served_round_once_and_frees_the_slot(
|
||||
gateway: Gateway, upstream: StreamedUpstream
|
||||
) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
|
||||
with _rig(
|
||||
gateway,
|
||||
"responses",
|
||||
upstream=upstream,
|
||||
billing=billing,
|
||||
prices=PRICES,
|
||||
slow_tool_seconds=SLOW_TOOL_SECONDS,
|
||||
) as rig:
|
||||
key: Final = _limited_key(rig)
|
||||
with rig.gateway.client.stream(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
headers={"Authorization": f"Bearer {key}"},
|
||||
json={"model": rig.model, "input": f"add {rig.alias}", "tools": [AUTO], "stream": True},
|
||||
timeout=90,
|
||||
) as hung_up:
|
||||
assert hung_up.status_code == 200
|
||||
call_id: Final = hung_up.headers["x-litellm-call-id"]
|
||||
eventually(lambda: _peer_add_calls(rig.peer), lambda found: len(found) == 1, seconds=30)
|
||||
eventually(lambda: _answers_directly(rig, key), lambda status: status == 200, seconds=60)
|
||||
posted: Final = tuple(
|
||||
JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST"
|
||||
)
|
||||
served: Final = tuple(
|
||||
FOLLOW_UP_ROUND if _has_tool_result(body) else FIRST_ROUND for body in posted if DIRECT not in _prompt(body)
|
||||
)
|
||||
expected: Final = _price(*served) + FIXED_MARGIN
|
||||
rows: Final = _model_rows(call_id, _bills_at_least(expected))
|
||||
assert served and rows, (served, rows)
|
||||
assert _billed(rows) == pytest.approx(expected), (served, rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("upstream", STREAMED_UPSTREAMS, ids=STREAMED_IDS)
|
||||
def test_streaming_responses_repeated_from_the_response_cache_bills_nothing(
|
||||
gateway: Gateway, upstream: StreamedUpstream
|
||||
) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
|
||||
with _rig(gateway, "responses", upstream=upstream, billing=billing, prices=PRICES) as rig:
|
||||
key: Final = _granted_key(rig)
|
||||
paid: Final = rig.send(key, [AUTO], stream=True)
|
||||
assert paid.status_code == 200 and rig.final_text(_completed(paid)) == ANSWER, paid.text
|
||||
paid_price: Final = _price(FIRST_ROUND, FOLLOW_UP_ROUND) + FIXED_MARGIN
|
||||
_model_rows(paid.headers["x-litellm-call-id"], _bills_at_least(paid_price))
|
||||
assert len(rig.upstream_tools()) == 2
|
||||
cached: Final = rig.send(key, [AUTO], stream=True)
|
||||
assert cached.status_code == 200 and rig.final_text(_completed(cached)) == ANSWER, cached.text
|
||||
assert rig.upstream_tools() == ()
|
||||
rows: Final = _model_rows(
|
||||
cached.headers["x-litellm-call-id"], lambda found: "success" in {row["status"] for row in found}
|
||||
)
|
||||
assert "success" in {row["status"] for row in rows} and _billed(rows) == 0.0, rows
|
||||
|
||||
|
||||
def test_bridged_next_turn_after_a_streamed_mcp_turn_replays_its_answer(gateway: Gateway) -> None:
|
||||
billing: Final = Billing(FIRST_ROUND, FOLLOW_UP_ROUND)
|
||||
with _rig(gateway, "responses", upstream="responses_bridge", billing=billing, prices=PRICES) as rig:
|
||||
key: Final = _granted_key(rig)
|
||||
first: Final = rig.send(key, [AUTO], stream=True)
|
||||
assert first.status_code == 200, first.text
|
||||
first_id: Final = string_value(_completed(first)["id"])
|
||||
_model_rows(first.headers["x-litellm-call-id"], _bills_at_least(_price(FIRST_ROUND, FOLLOW_UP_ROUND)))
|
||||
rig.wire.drain()
|
||||
second: Final = rig.send(key, [AUTO], previous_response_id=first_id, cache={"no-cache": True})
|
||||
assert second.status_code == 200, second.text
|
||||
opening: Final = next(
|
||||
JSON_OBJECT.validate_json(request.body) for request in rig.wire.drain() if request.method == "POST"
|
||||
)
|
||||
assert ANSWER in _assistant_texts(opening), opening
|
||||
|
||||
|
||||
Bridge = Literal["chat", "responses", "messages"]
|
||||
BRIDGES: Final[tuple[Bridge, ...]] = ("chat", "responses")
|
||||
Client = Literal["sync", "async"]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,418 @@
|
|||
import json
|
||||
import math
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.process import OwnedProxy, WorkerHealthcheck, graceful_stop_seconds, owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
Route = Literal["/v1/extraction", "/extraction"]
|
||||
|
||||
_API_KEY: Final = "synthetic-cloudflare-key"
|
||||
_CLAIRIC: Final = "cloudflare/@cf/cloudflare/clairic"
|
||||
_CLAIRIC_PATH: Final = "/ai/run/@cf/cloudflare/clairic"
|
||||
_INPUT_TOKENS: Final = 412
|
||||
_EMAIL: Final = "billing@acmetools.example"
|
||||
_SCENARIO: Final = re.compile(r"scenario-([a-z-]+)")
|
||||
_JSON_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
_FIELDS: Final[dict[str, JsonValue]] = {
|
||||
"vendor": {"type": "text", "instructions": "Who sent the invoice?"},
|
||||
"total": {"type": "number", "unit": "USD"},
|
||||
}
|
||||
_SPEND_QUERY: Final = (
|
||||
'SELECT spend, status, call_type, model_group, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" '
|
||||
"WHERE request_id = %s"
|
||||
)
|
||||
_OWNED_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120)
|
||||
_REQUEST_GUARDRAIL: Final = "extraction-mask-email-in"
|
||||
_RESPONSE_GUARDRAIL: Final = "extraction-mask-email-out"
|
||||
_OWNED_MODEL: Final = "invoice-reader"
|
||||
|
||||
|
||||
def _state(marker: str, scenario: str = "plain") -> str:
|
||||
return f"Invoice from Acme Tools, total $174. scenario-{scenario} marker-{marker}"
|
||||
|
||||
|
||||
def _vendor_span(state: str) -> list[JsonValue]:
|
||||
start: Final = state.index("Acme Tools")
|
||||
return [start, start + len("Acme Tools")]
|
||||
|
||||
|
||||
def _email_span(state: str) -> list[JsonValue]:
|
||||
start: Final = state.index(_EMAIL)
|
||||
return [start, start + len(_EMAIL)]
|
||||
|
||||
|
||||
def _answers(state: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)},
|
||||
"total": {"type": "number", "value": 174, "confidence": 0.95},
|
||||
}
|
||||
|
||||
|
||||
def _envelope(result: Mapping[str, JsonValue]) -> Reply:
|
||||
return Reply(body=json.dumps({"result": result, "success": True, "errors": [], "messages": []}).encode())
|
||||
|
||||
|
||||
def _clairic_reply(request: Request) -> Reply:
|
||||
state: Final = JSON_OBJECT.validate_json(request.body)["state"]
|
||||
assert isinstance(state, str), request.body
|
||||
return _envelope(
|
||||
{
|
||||
"model": "clairic",
|
||||
"answers": _answers(state),
|
||||
"truncated": False,
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _expected_response(model: str, state: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": model,
|
||||
"answers": _answers(state),
|
||||
"truncated": False,
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
}
|
||||
|
||||
|
||||
def _upstream_body(request: Request) -> dict[str, JsonValue]:
|
||||
assert (request.method, request.target) == ("POST", _CLAIRIC_PATH), (request.method, request.target)
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}", request.headers
|
||||
return JSON_OBJECT.validate_json(request.body)
|
||||
|
||||
|
||||
def _body(model: str, marker: str, **extra: JsonValue) -> dict[str, JsonValue]:
|
||||
return {"model": model, "state": _state(marker), "fields": _FIELDS, **extra}
|
||||
|
||||
|
||||
def _extract(
|
||||
gateway: Gateway,
|
||||
model: str,
|
||||
marker: str,
|
||||
*,
|
||||
route: Route = "/v1/extraction",
|
||||
key: str | None = None,
|
||||
**extra: JsonValue,
|
||||
) -> httpx.Response:
|
||||
return gateway.request("POST", route, _body(model, marker, **extra), key=key)
|
||||
|
||||
|
||||
def _number(value: JsonValue) -> float:
|
||||
assert isinstance(value, (int, float)) and not isinstance(value, bool), value
|
||||
return float(value)
|
||||
|
||||
|
||||
def _expected_spend() -> float:
|
||||
cost_map: Final = JSON_OBJECT.validate_json(Path("model_prices_and_context_window.json").read_bytes())
|
||||
priced: Final = cost_map.get(_CLAIRIC)
|
||||
assert priced is not None, f"{_CLAIRIC} has no cost map entry"
|
||||
return _INPUT_TOKENS * _number(object_value(priced)["input_cost_per_token"])
|
||||
|
||||
|
||||
def _spend_row(call_id: str) -> dict[str, JsonValue]:
|
||||
rows: Final = eventually(lambda: read_rows(_SPEND_QUERY, (call_id,)), lambda found: len(found) == 1, seconds=70)
|
||||
return rows[0]
|
||||
|
||||
|
||||
def _assert_billed(gateway: Gateway, response: httpx.Response, model: str) -> None:
|
||||
expected: Final = _expected_spend()
|
||||
assert math.isclose(float(response.headers["x-litellm-response-cost"]), expected, rel_tol=1e-9), response.headers
|
||||
call_id: Final = response.headers["x-litellm-call-id"]
|
||||
row: Final = _spend_row(call_id)
|
||||
assert (row["status"], row["call_type"], row["model_group"], row["prompt_tokens"], row["completion_tokens"]) == (
|
||||
"success",
|
||||
"aextraction",
|
||||
model,
|
||||
_INPUT_TOKENS,
|
||||
0,
|
||||
), row
|
||||
assert math.isclose(_number(row["spend"]), expected, rel_tol=1e-9), row
|
||||
listed: Final = gateway.request("GET", "/spend/logs", params={"request_id": call_id})
|
||||
assert listed.status_code == 200, listed.text
|
||||
(reported,) = _JSON_ROWS.validate_json(listed.content)
|
||||
assert (reported["request_id"], reported["model_group"]) == (call_id, model), reported
|
||||
assert math.isclose(_number(reported["spend"]), expected, rel_tol=1e-9), reported
|
||||
|
||||
|
||||
@pytest.mark.parametrize("route", ("/v1/extraction", "/extraction"))
|
||||
def test_extraction_reaches_clairic_on_both_routes_and_is_billed(gateway: Gateway, route: Route) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _extract(gateway, model, marker, route=route, metadata={"tags": [f"audit-{marker}"]})
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response(model, _state(marker))
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS}
|
||||
_assert_billed(gateway, response, model)
|
||||
|
||||
|
||||
_INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = (
|
||||
("unknown_type", {"fields": {"vendor": {"type": "paragraph"}}}),
|
||||
("list_without_items", {"fields": {"items": {"type": "list"}}}),
|
||||
("remote_image", {"images": ["https://images.example/receipt.png"]}),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("label", "change"), _INVALID_BODIES, ids=[label for label, _ in _INVALID_BODIES])
|
||||
def test_invalid_extraction_bodies_are_refused_before_any_upstream_call(
|
||||
gateway: Gateway, label: str, change: dict[str, JsonValue]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/extraction", {**_body(model, marker), **change})
|
||||
assert response.status_code == 400, (label, response.text)
|
||||
assert "Invalid Extraction request" in response.text, (label, response.text)
|
||||
assert wire.drain() == (), label
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("stream", True)), ids=("temperature", "stream"))
|
||||
def test_unsupported_top_level_params_are_refused_unless_the_deployment_drops_params(
|
||||
gateway: Gateway, param: str, value: JsonValue
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
|
||||
strict: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
dropping: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY, drop_params=True)
|
||||
refused: Final = _extract(gateway, strict, marker, **{param: value})
|
||||
assert refused.status_code == 400, refused.text
|
||||
assert param in refused.text, refused.text
|
||||
assert wire.drain() == ()
|
||||
dropped: Final = _extract(gateway, dropping, marker, **{param: value})
|
||||
assert dropped.status_code == 200, dropped.text
|
||||
assert dropped.headers["content-type"].startswith("application/json"), dropped.headers
|
||||
assert dropped.json() == _expected_response(dropping, _state(marker))
|
||||
(request,) = wire.drain()
|
||||
assert _upstream_body(request) == {"model": "clairic", "state": _state(marker), "fields": _FIELDS}
|
||||
_assert_billed(gateway, dropped, dropping)
|
||||
|
||||
|
||||
def test_key_checks_on_extraction_match_chat(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
anonymous: Final = gateway.client.post("/v1/extraction", json=_body(model, marker))
|
||||
assert anonymous.status_code == 401, anonymous.text
|
||||
restricted: Final = scenario.key(models=[f"other-{uuid.uuid4().hex}"])
|
||||
refused: Final = _extract(gateway, model, marker, key=restricted)
|
||||
chat_refused: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": marker}]},
|
||||
key=restricted,
|
||||
)
|
||||
assert 400 <= refused.status_code < 500, refused.text
|
||||
assert refused.status_code == chat_refused.status_code, (refused.text, chat_refused.text)
|
||||
assert wire.drain() == ()
|
||||
allowed: Final = scenario.key(models=[model], max_budget=1e-06)
|
||||
first: Final = _extract(gateway, model, marker, key=allowed)
|
||||
assert first.status_code == 200, first.text
|
||||
assert len(wire.drain()) == 1
|
||||
blocked, upstream_calls = eventually(
|
||||
lambda: (_extract(gateway, model, uuid.uuid4().hex, key=allowed), len(wire.drain())),
|
||||
lambda answer: answer[0].status_code != 200,
|
||||
seconds=70,
|
||||
)
|
||||
chat_blocked: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": marker}]},
|
||||
key=allowed,
|
||||
)
|
||||
assert 400 <= blocked.status_code < 500, blocked.text
|
||||
assert blocked.status_code == chat_blocked.status_code, (blocked.text, chat_blocked.text)
|
||||
assert upstream_calls == 0
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def test_a_small_key_tpm_limit_serves_extraction_and_counts_its_input_tokens(gateway: Gateway) -> None:
|
||||
with wire_server(_clairic_reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
key: Final = scenario.key(models=[model], tpm_limit=100)
|
||||
first: Final = _extract(gateway, model, uuid.uuid4().hex, key=key)
|
||||
assert first.status_code == 200, first.text
|
||||
assert len(wire.drain()) == 1
|
||||
limited, upstream_calls = eventually(
|
||||
lambda: (_extract(gateway, model, uuid.uuid4().hex, key=key), len(wire.drain())),
|
||||
lambda answer: answer[0].status_code != 200,
|
||||
seconds=70,
|
||||
)
|
||||
assert limited.status_code == 429, limited.text
|
||||
assert upstream_calls == 0
|
||||
|
||||
|
||||
def test_cloudflare_errors_keep_their_status_and_log_an_unbilled_failure(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
refusal: Final = json.dumps(
|
||||
{
|
||||
"result": None,
|
||||
"success": False,
|
||||
"errors": [{"code": 5006, "message": f"min above max {marker}"}],
|
||||
"messages": [],
|
||||
}
|
||||
).encode()
|
||||
with wire_server(lambda _: Reply(status=422, body=refusal)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_CLAIRIC, api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = _extract(gateway, model, marker)
|
||||
assert response.status_code == 422, response.text
|
||||
assert f"min above max {marker}" in response.text, response.text
|
||||
assert len(wire.drain()) >= 1
|
||||
row: Final = _spend_row(response.headers["x-litellm-call-id"])
|
||||
assert (row["status"], row["call_type"], row["model_group"], _number(row["spend"])) == (
|
||||
"failure",
|
||||
"aextraction",
|
||||
model,
|
||||
0.0,
|
||||
), row
|
||||
|
||||
|
||||
def _owned_reply(request: Request) -> Reply:
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
state: Final = body["state"]
|
||||
assert isinstance(state, str), request.body
|
||||
scenario: Final = _SCENARIO.search(state)
|
||||
assert scenario is not None, state
|
||||
if scenario.group(1) != "mask-out":
|
||||
return _clairic_reply(request)
|
||||
return _envelope(
|
||||
{
|
||||
"model": "clairic",
|
||||
"answers": {
|
||||
"vendor": {"type": "text", "value": "Acme Tools", "confidence": 0.97, "span": _vendor_span(state)},
|
||||
"contact_email": {"type": "text", "value": _EMAIL, "confidence": 0.9, "span": _email_span(state)},
|
||||
},
|
||||
"truncated": False,
|
||||
"state_text": state,
|
||||
"usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": 0},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _owned_config(upstream_url: str) -> dict[str, JsonValue]:
|
||||
base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
|
||||
email_filter: Final[list[JsonValue]] = [{"pattern_type": "prebuilt", "pattern_name": "email", "action": "MASK"}]
|
||||
return {
|
||||
**base,
|
||||
"guardrails": [
|
||||
{
|
||||
"guardrail_name": _REQUEST_GUARDRAIL,
|
||||
"litellm_params": {"guardrail": "litellm_content_filter", "mode": "pre_call", "patterns": email_filter},
|
||||
},
|
||||
{
|
||||
"guardrail_name": _RESPONSE_GUARDRAIL,
|
||||
"litellm_params": {
|
||||
"guardrail": "litellm_content_filter",
|
||||
"mode": "post_call",
|
||||
"patterns": email_filter,
|
||||
},
|
||||
},
|
||||
],
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": _OWNED_MODEL,
|
||||
"litellm_params": {"model": _CLAIRIC, "api_base": upstream_url, "api_key": _API_KEY},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rig:
|
||||
proxy: OwnedProxy
|
||||
upstream: Wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
|
||||
with wire_server(_owned_reply) as upstream:
|
||||
directory: Final = tmp_path_factory.mktemp("clairic-extraction-owned")
|
||||
config_path: Final = directory / "clairic-extraction.yaml"
|
||||
config_path.write_text(yaml.safe_dump(_owned_config(upstream.url)))
|
||||
with owned_proxy_process(
|
||||
session_gateway,
|
||||
directory,
|
||||
{},
|
||||
config=config_path,
|
||||
workers=2,
|
||||
extra_arguments=("--max_tokens", "256", "--temperature", "0.2"),
|
||||
healthcheck=WorkerHealthcheck.LENIENT,
|
||||
) as proxy:
|
||||
yield Rig(proxy, upstream)
|
||||
|
||||
|
||||
def _owned_extract(rig: Rig, state: str, **extra: JsonValue) -> httpx.Response:
|
||||
return rig.proxy.gateway.request(
|
||||
"POST", "/v1/extraction", {"model": _OWNED_MODEL, "state": state, "fields": _FIELDS, **extra}
|
||||
)
|
||||
|
||||
|
||||
def _applied_guardrails(response: httpx.Response) -> frozenset[str]:
|
||||
return frozenset(name.strip() for name in response.headers.get("x-litellm-applied-guardrails", "").split(","))
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_cli_generation_defaults_never_reach_an_extraction_call(rig: Rig) -> None:
|
||||
state: Final = _state(uuid.uuid4().hex)
|
||||
response: Final = _owned_extract(rig, state)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _expected_response(_OWNED_MODEL, state)
|
||||
(request,) = rig.upstream.drain()
|
||||
assert _upstream_body(request) == {"model": "clairic", "state": state, "fields": _FIELDS}
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_a_request_guardrail_masks_the_state_before_clairic_and_drops_spans_into_it(rig: Rig) -> None:
|
||||
state: Final = f"{_state(uuid.uuid4().hex, 'mask-in')} Questions go to {_EMAIL}."
|
||||
response: Final = _owned_extract(rig, state, guardrails=[_REQUEST_GUARDRAIL])
|
||||
assert response.status_code == 200, response.text
|
||||
assert _REQUEST_GUARDRAIL in _applied_guardrails(response), response.headers
|
||||
(request,) = rig.upstream.drain()
|
||||
sent: Final = _upstream_body(request)
|
||||
sent_state: Final = sent["state"]
|
||||
assert isinstance(sent_state, str) and _EMAIL not in sent_state, sent
|
||||
assert sent_state.startswith(state.removesuffix(f"{_EMAIL}.")) and "REDACTED" in sent_state, sent
|
||||
assert sent == {"model": "clairic", "state": sent_state, "fields": _FIELDS}
|
||||
answers: Final = object_value(response.json()["answers"])
|
||||
assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers
|
||||
|
||||
|
||||
@pytest.mark.timeout(_OWNED_TIMEOUT)
|
||||
def test_a_response_guardrail_masks_text_answers_and_state_text(rig: Rig) -> None:
|
||||
state: Final = f"{_state(uuid.uuid4().hex, 'mask-out')} Questions go to {_EMAIL}."
|
||||
fields: Final[dict[str, JsonValue]] = {**_FIELDS, "contact_email": {"type": "text"}}
|
||||
response: Final = rig.proxy.gateway.request(
|
||||
"POST",
|
||||
"/v1/extraction",
|
||||
{
|
||||
"model": _OWNED_MODEL,
|
||||
"state": state,
|
||||
"fields": fields,
|
||||
"guardrails": [_RESPONSE_GUARDRAIL],
|
||||
"options": {"return_state_text": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _RESPONSE_GUARDRAIL in _applied_guardrails(response), response.headers
|
||||
(request,) = rig.upstream.drain()
|
||||
assert _upstream_body(request)["state"] == state
|
||||
body: Final = response.json()
|
||||
state_text: Final = body["state_text"]
|
||||
assert isinstance(state_text, str) and _EMAIL not in state_text and "REDACTED" in state_text, body
|
||||
answers: Final = object_value(body["answers"])
|
||||
contact: Final = object_value(answers["contact_email"])
|
||||
assert "span" not in contact, answers
|
||||
masked: Final = contact["value"]
|
||||
assert isinstance(masked, str) and _EMAIL not in masked and "REDACTED" in masked, answers
|
||||
assert object_value(answers["vendor"]) == {"type": "text", "value": "Acme Tools", "confidence": 0.97}, answers
|
||||
|
|
@ -1,16 +1,113 @@
|
|||
import json
|
||||
import os
|
||||
import uuid
|
||||
from itertools import product
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway, eventually, object_value
|
||||
from integration._support.process import graceful_stop_seconds, owned_proxy
|
||||
from integration._support.redis_process import owned_redis
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from redis import Redis
|
||||
|
||||
import litellm
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.caching.redis_cache import RedisCache
|
||||
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY
|
||||
from litellm.constants import GLOBAL_PROXY_SPEND_CACHE_KEY, LITELLM_PROXY_BUDGET_NAME
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.proxy_server import update_cache
|
||||
|
||||
BUDGET_MODEL: Final = "proxy-budget-model"
|
||||
CONFIG: Final = TypeAdapter(dict[str, JsonValue])
|
||||
SECTION: Final = TypeAdapter(dict[str, JsonValue])
|
||||
OWNED_PROXY_SECONDS: Final = 2 * graceful_stop_seconds() + 120
|
||||
|
||||
|
||||
def _proxy_budget_config(gateway: Gateway, directory: Path, max_budget: float) -> Path:
|
||||
loaded: Final = CONFIG.validate_python(
|
||||
yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text())
|
||||
)
|
||||
model: Final[JsonValue] = {
|
||||
"model_name": BUDGET_MODEL,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": f"{gateway.upstream_url}/v1",
|
||||
"input_cost_per_token": 0.001,
|
||||
"output_cost_per_token": 0.002,
|
||||
},
|
||||
}
|
||||
path: Final = directory / "proxy_budget_config.yaml"
|
||||
path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
**loaded,
|
||||
"model_list": [model],
|
||||
"general_settings": {
|
||||
**SECTION.validate_python(loaded["general_settings"]),
|
||||
"user_api_key_cache_ttl": 600,
|
||||
},
|
||||
"litellm_settings": {
|
||||
**SECTION.validate_python(loaded["litellm_settings"]),
|
||||
"max_budget": max_budget,
|
||||
"budget_duration": "30d",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _chat(proxy: Gateway, key: str, marker: str) -> httpx.Response:
|
||||
return proxy.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": BUDGET_MODEL, "max_tokens": 20, "messages": [{"role": "user", "content": marker}]},
|
||||
key=key,
|
||||
)
|
||||
|
||||
|
||||
def _served_id(proxy: Gateway, key: str) -> str:
|
||||
response: Final = _chat(proxy, key, f"proxy budget {uuid.uuid4().hex}")
|
||||
assert response.status_code == 200, response.text
|
||||
identity: Final = response.json()["id"]
|
||||
assert isinstance(identity, str), response.text
|
||||
return identity
|
||||
|
||||
|
||||
def _logged_spend(proxy: Gateway, request_ids: tuple[str, ...]) -> tuple[float, ...]:
|
||||
rows: Final = eventually(
|
||||
lambda: proxy.rows(
|
||||
'SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = ANY(%s::text[])',
|
||||
("{" + ",".join(request_ids) + "}",),
|
||||
),
|
||||
lambda found: len(found) == len(request_ids),
|
||||
seconds=70,
|
||||
)
|
||||
return tuple(float(str(row["spend"])) for row in rows)
|
||||
|
||||
|
||||
def _proxy_budget_spend(proxy: Gateway) -> float | None:
|
||||
rows: Final = proxy.rows('SELECT spend FROM "LiteLLM_UserTable" WHERE user_id = %s', (LITELLM_PROXY_BUDGET_NAME,))
|
||||
return float(str(rows[0]["spend"])) if rows else None
|
||||
|
||||
|
||||
def _counter_key(redis: Redis) -> str:
|
||||
found: Final = tuple(str(key, "utf-8") for key in redis.scan_iter(match=f"*{GLOBAL_PROXY_SPEND_CACHE_KEY}"))
|
||||
assert len(found) == 1, found
|
||||
return found[0]
|
||||
|
||||
|
||||
def _upstream_saw(proxy: Gateway, marker: str) -> bool:
|
||||
with httpx.Client(base_url=proxy.upstream_url, timeout=5, trust_env=False) as upstream:
|
||||
response: Final = upstream.get("/__observations")
|
||||
response.raise_for_status()
|
||||
return marker in json.dumps(object_value(response.json())["requests"])
|
||||
|
||||
|
||||
async def _add_request_cost(cache: UserApiKeyCache, response_cost: float) -> None:
|
||||
await update_cache(
|
||||
|
|
@ -56,3 +153,63 @@ async def test_global_proxy_spend_adds_every_instance_cost_and_stays_cleared_aft
|
|||
assert await redis_cache.async_get_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY) is None
|
||||
finally:
|
||||
await redis_cache.async_delete_cache(key=GLOBAL_PROXY_SPEND_CACHE_KEY)
|
||||
|
||||
|
||||
@pytest.mark.timeout(OWNED_PROXY_SECONDS)
|
||||
def test_a_key_with_no_user_spends_toward_the_proxy_max_budget(gateway: Gateway, tmp_path: Path) -> None:
|
||||
with (
|
||||
owned_proxy(gateway, tmp_path, {}, config=_proxy_budget_config(gateway, tmp_path, max_budget=0.01)) as owned,
|
||||
owned.scenario() as scenario,
|
||||
):
|
||||
key: Final = scenario.key(models=[BUDGET_MODEL])
|
||||
(spent,) = _logged_spend(owned, (_served_id(owned, key),))
|
||||
assert spent > 0.01, spent
|
||||
eventually(
|
||||
lambda: _proxy_budget_spend(owned),
|
||||
lambda recorded: recorded is not None and recorded > 0.01,
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
marker: Final = f"over the proxy budget {uuid.uuid4().hex}"
|
||||
refused: Final = _chat(owned, key, marker)
|
||||
|
||||
assert refused.status_code != 200, refused.text
|
||||
assert object_value(refused.json()["error"])["type"] == "budget_exceeded", refused.text
|
||||
assert not _upstream_saw(owned, marker), refused.text
|
||||
|
||||
|
||||
@pytest.mark.timeout(2 * OWNED_PROXY_SECONDS)
|
||||
def test_two_proxies_add_every_request_to_one_proxy_spend_counter(gateway: Gateway, tmp_path: Path) -> None:
|
||||
config: Final = _proxy_budget_config(gateway, tmp_path, max_budget=1000.0)
|
||||
with owned_redis(tmp_path / "redis") as cache, Redis(host=cache.host, port=cache.port) as redis:
|
||||
shared_redis: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)}
|
||||
with owned_proxy(gateway, tmp_path / "first", shared_redis, config=config) as first:
|
||||
assert first.database_url is not None
|
||||
with (
|
||||
owned_proxy(
|
||||
gateway, tmp_path / "second", {**shared_redis, "DATABASE_URL": first.database_url}, config=config
|
||||
) as second,
|
||||
first.scenario() as scenario,
|
||||
):
|
||||
user_key: Final = scenario.key(user_id=scenario.user(), models=[BUDGET_MODEL])
|
||||
userless_key: Final = scenario.key(models=[BUDGET_MODEL])
|
||||
counter: Final = _counter_key(redis)
|
||||
ttl_before: Final = redis.ttl(counter)
|
||||
|
||||
request_ids: Final = tuple(
|
||||
_served_id(proxy, key) for key, proxy in product((user_key, userless_key), (first, second))
|
||||
)
|
||||
total: Final = sum(_logged_spend(first, request_ids))
|
||||
persisted: Final = eventually(
|
||||
lambda: _proxy_budget_spend(first),
|
||||
lambda spend: spend is not None and spend == pytest.approx(total),
|
||||
seconds=70,
|
||||
)
|
||||
|
||||
assert persisted == pytest.approx(total)
|
||||
eventually(
|
||||
lambda: redis.get(counter),
|
||||
lambda shared: shared is not None and float(str(shared, "utf-8")) == pytest.approx(total),
|
||||
seconds=70,
|
||||
)
|
||||
assert 0 < redis.ttl(counter) <= ttl_before
|
||||
|
|
|
|||
|
|
@ -105,6 +105,24 @@ def test_openai_image_edit_sync_accepts_bytesio_images(respx_mock: respx.MockRou
|
|||
assert _multipart_image_parts(route.calls[0].request) == (_FIRST_IMAGE,)
|
||||
|
||||
|
||||
def test_openai_image_edit_without_a_model_leaves_model_out_of_the_request(
|
||||
respx_mock: respx.MockRouter, httpx_transport: None
|
||||
) -> None:
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock(
|
||||
return_value=httpx.Response(200, json=_EDIT_RESPONSE)
|
||||
)
|
||||
|
||||
litellm.image_edit(
|
||||
prompt="combine the reference images",
|
||||
image=io.BytesIO(_FIRST_IMAGE),
|
||||
custom_llm_provider="openai",
|
||||
api_key="fake-key",
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert b'name="model"' not in route.calls[0].request.read()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_image_edit_accepts_bytesio_images(respx_mock: respx.MockRouter, httpx_transport: None) -> None:
|
||||
route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock(
|
||||
|
|
|
|||
|
|
@ -300,7 +300,7 @@ async def test_aimage_generation_ignores_a_non_string_base_url_like_the_sync_pat
|
|||
)
|
||||
|
||||
response: Final = await litellm.aimage_generation(
|
||||
model="dall-e-3",
|
||||
model="gpt-image-2.5-sunburst",
|
||||
prompt="a red circle",
|
||||
api_base=api_base,
|
||||
base_url=httpx.URL(api_base),
|
||||
|
|
@ -688,3 +688,24 @@ async def test_chatgpt_images_forward_client_login_and_workspace(
|
|||
assert "cookie" not in request.headers
|
||||
assert "x-litellm-api-key" not in request.headers
|
||||
assert json.loads(request.content) == {"model": "gpt-image-2", "prompt": "A blue circle"}
|
||||
|
||||
|
||||
def test_image_generation_without_a_model_or_provider_fails_before_calling_any_provider(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="model is required"):
|
||||
litellm.image_generation(prompt="a red circle", api_key="fake_openai_api_key")
|
||||
|
||||
assert not respx_mock.calls
|
||||
|
||||
|
||||
def test_topaz_image_variation_works_without_a_model(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post("https://api.topazlabs.com/image/v1/enhance").mock(
|
||||
return_value=httpx.Response(200, content=b"enhanced-image", headers={"content-type": "image/png"})
|
||||
)
|
||||
|
||||
response: Final = litellm.image_variation(image=b"source-image", custom_llm_provider="topaz", api_key="fake-key")
|
||||
|
||||
assert route.call_count == 1
|
||||
assert response.data is not None
|
||||
assert response.data[0].b64_json == base64.b64encode(b"enhanced-image").decode()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from typing import Dict, Final, List
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.router_utils.batch_utils import (
|
||||
from litellm.litellm_core_utils.batch_utils import (
|
||||
InMemoryFile,
|
||||
get_router_metadata_variable_name,
|
||||
parse_jsonl_with_embedded_newlines,
|
||||
|
|
@ -116,7 +116,7 @@ def test_router_metadata_variable_name():
|
|||
|
||||
def test_non_json_input():
|
||||
"""Test that replace_model_in_jsonl returns original content for non-JSON input"""
|
||||
from litellm.router_utils.batch_utils import replace_model_in_jsonl
|
||||
from litellm.litellm_core_utils.batch_utils import replace_model_in_jsonl
|
||||
|
||||
# Test with non-JSON string
|
||||
non_json_str = "This is not a JSON string"
|
||||
|
|
@ -138,7 +138,7 @@ def test_non_json_input():
|
|||
|
||||
def test_should_replace_model_in_jsonl():
|
||||
"""Test that should_replace_model_in_jsonl returns the correct value"""
|
||||
from litellm.router_utils.batch_utils import should_replace_model_in_jsonl
|
||||
from litellm.litellm_core_utils.batch_utils import should_replace_model_in_jsonl
|
||||
|
||||
assert should_replace_model_in_jsonl(purpose="batch") is True
|
||||
assert should_replace_model_in_jsonl(purpose="batch", passthrough=True) is False
|
||||
|
|
@ -340,7 +340,7 @@ def test_replace_model_in_jsonl_with_embedded_newlines():
|
|||
|
||||
|
||||
def test_is_batch_retrieve_call_type_matches_only_batch_retrieves():
|
||||
from litellm.router_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.litellm_core_utils.batch_utils import is_batch_retrieve_call_type
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
assert is_batch_retrieve_call_type(CallTypes.aretrieve_batch.value) is True
|
||||
|
|
@ -389,7 +389,8 @@ def setup_and_teardown():
|
|||
)
|
||||
def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str, cost: float, eligible: bool) -> None:
|
||||
from openai.types import Batch
|
||||
from litellm.router_utils.batch_utils import batch_id_to_charge_once
|
||||
|
||||
from litellm.litellm_core_utils.batch_utils import batch_id_to_charge_once
|
||||
|
||||
batch: Final = Batch(
|
||||
id="batch-budget", completion_window="24h", created_at=1, endpoint="/v1/chat/completions",
|
||||
|
|
@ -401,7 +402,7 @@ def test_batch_charge_identity_requires_a_cost_bearing_retrieval(call_type: str,
|
|||
|
||||
def test_batch_charge_markers_keep_counters_and_batches_independent() -> None:
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.router_utils.batch_utils import batch_charged_once_marker_key
|
||||
from litellm.litellm_core_utils.batch_utils import batch_charged_once_marker_key
|
||||
|
||||
cache: Final = InMemoryCache()
|
||||
cache.set_cache(batch_charged_once_marker_key("provider-spend", "first-batch"), 1)
|
||||
|
|
@ -20,27 +20,9 @@ from litellm.llms.azure.image_generation.http_utils import (
|
|||
azure_deployment_image_generation_json_body,
|
||||
)
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.azure.image_generation import (
|
||||
AzureDallE3ImageGenerationConfig,
|
||||
get_azure_image_generation_config,
|
||||
)
|
||||
from litellm.utils import get_optional_params_image_gen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"received_model, expected_config",
|
||||
[
|
||||
("dall-e-3", AzureDallE3ImageGenerationConfig),
|
||||
("dalle-3", AzureDallE3ImageGenerationConfig),
|
||||
("openai_dall_e_3", AzureDallE3ImageGenerationConfig),
|
||||
],
|
||||
)
|
||||
def test_azure_image_generation_config(received_model, expected_config):
|
||||
assert isinstance(
|
||||
get_azure_image_generation_config(received_model), expected_config
|
||||
)
|
||||
|
||||
|
||||
def test_azure_deployment_image_generation_json_body():
|
||||
"""Deployment-scoped Azure image URL must not send ``model`` in JSON."""
|
||||
api = (
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ class TestImageGenerationEntryPointHeaders:
|
|||
test_headers = {"cf-aig-authorization": "Bearer my-secret"}
|
||||
|
||||
await litellm.aimage_generation(
|
||||
model="dall-e-3",
|
||||
model="gpt-image-2.5-sunburst",
|
||||
prompt="A white cat",
|
||||
extra_headers=test_headers,
|
||||
client=mock_openai_client,
|
||||
|
|
|
|||
|
|
@ -46,12 +46,12 @@ API_FUNCTION_PARAMS = [
|
|||
(
|
||||
"image_generation",
|
||||
False,
|
||||
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
|
||||
{"model": "gpt-image-2.5-sunburst", "prompt": "A beautiful sunset over mountains"},
|
||||
),
|
||||
(
|
||||
"image_generation",
|
||||
True,
|
||||
{"model": "dall-e-3", "prompt": "A beautiful sunset over mountains"},
|
||||
{"model": "gpt-image-2.5-sunburst", "prompt": "A beautiful sunset over mountains"},
|
||||
),
|
||||
(
|
||||
"speech",
|
||||
|
|
@ -84,11 +84,7 @@ async def test_openai_client_reuse(function_name, is_async, args):
|
|||
"""
|
||||
|
||||
# Determine which client class to mock based on whether the test is async
|
||||
client_path = (
|
||||
"litellm.llms.openai.openai.AsyncOpenAI"
|
||||
if is_async
|
||||
else "litellm.llms.openai.openai.OpenAI"
|
||||
)
|
||||
client_path = "litellm.llms.openai.openai.AsyncOpenAI" if is_async else "litellm.llms.openai.openai.OpenAI"
|
||||
|
||||
# Create the appropriate patches
|
||||
with (
|
||||
|
|
@ -98,9 +94,7 @@ async def test_openai_client_reuse(function_name, is_async, args):
|
|||
):
|
||||
# Setup the mock to return None first time (cache miss) then a client for subsequent calls
|
||||
mock_client = MagicMock()
|
||||
mock_get_cache.side_effect = [None] + [
|
||||
mock_client
|
||||
] * 9 # First call returns None, rest return the mock client
|
||||
mock_get_cache.side_effect = [None] + [mock_client] * 9 # First call returns None, rest return the mock client
|
||||
|
||||
# Make 10 API calls
|
||||
for _ in range(10):
|
||||
|
|
@ -118,9 +112,9 @@ async def test_openai_client_reuse(function_name, is_async, args):
|
|||
pass
|
||||
|
||||
# Verify client was created only once
|
||||
assert (
|
||||
mock_client_class.call_count == 1
|
||||
), f"{'Async' if is_async else ''}OpenAI client should be created only once"
|
||||
assert mock_client_class.call_count == 1, (
|
||||
f"{'Async' if is_async else ''}OpenAI client should be created only once"
|
||||
)
|
||||
|
||||
# Verify the client was cached
|
||||
assert mock_set_cache.call_count == 1, "Client should be cached once"
|
||||
|
|
@ -144,12 +138,8 @@ def test_precomputed_init_params_match_inspect_signature():
|
|||
_OPENAI_INIT_PARAMS,
|
||||
)
|
||||
|
||||
expected_openai = tuple(
|
||||
p for p in inspect.signature(OpenAI.__init__).parameters if p != "self"
|
||||
)
|
||||
expected_azure = tuple(
|
||||
p for p in inspect.signature(AzureOpenAI.__init__).parameters if p != "self"
|
||||
)
|
||||
expected_openai = tuple(p for p in inspect.signature(OpenAI.__init__).parameters if p != "self")
|
||||
expected_azure = tuple(p for p in inspect.signature(AzureOpenAI.__init__).parameters if p != "self")
|
||||
|
||||
assert _OPENAI_INIT_PARAMS == expected_openai
|
||||
assert _AZURE_OPENAI_INIT_PARAMS == expected_azure
|
||||
|
|
@ -500,7 +490,10 @@ async def test_sdk_api_factories_keep_httpx_transport_and_request_timeouts(api:
|
|||
moved: Final = mock.get(host="sdk-default.example", path="/v1/moved-models").mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"object": "list", "data": [{"id": "sdk-compat", "object": "model", "created": 0, "owned_by": "t"}]},
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{"id": "sdk-compat", "object": "model", "created": 0, "owned_by": "t"}],
|
||||
},
|
||||
)
|
||||
)
|
||||
sdk_client: Final = _sdk_api_client(api, is_async, timeout)
|
||||
|
|
@ -542,7 +535,10 @@ async def test_azure_gateway_and_image_variations_use_the_callers_async_session(
|
|||
assert azure_client is not None
|
||||
assert azure_client._client is session
|
||||
response: Final = await aimage_variation(
|
||||
image=BytesIO(b"image-bytes"), api_key="transport-only", api_base="https://sdk-default.example/v1"
|
||||
image=BytesIO(b"image-bytes"),
|
||||
custom_llm_provider="openai",
|
||||
api_key="transport-only",
|
||||
api_base="https://sdk-default.example/v1",
|
||||
)
|
||||
assert response.data[0].url == "https://example.com/image.png"
|
||||
assert not session.is_closed
|
||||
|
|
|
|||
|
|
@ -6,12 +6,14 @@ from datetime import datetime, timedelta
|
|||
from queue import SimpleQueue
|
||||
from threading import get_ident
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.messages.transformation import (
|
||||
VertexAIPartnerModelsAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -94,6 +96,85 @@ async def test_messages_auth_load_and_refresh_leave_the_event_loop(stream: bool,
|
|||
assert headers == {"Authorization": "Bearer incoming", "anthropic-beta": "retained-beta", "x-request-id": "req-1"}
|
||||
|
||||
|
||||
def _vertex_client() -> MagicMock:
|
||||
vertex_response: Final = MagicMock()
|
||||
vertex_response.json.return_value = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"model": "claude-sonnet-5",
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
}
|
||||
client: Final = MagicMock(spec=AsyncHTTPHandler)
|
||||
client.post = AsyncMock(return_value=vertex_response)
|
||||
return client
|
||||
|
||||
|
||||
async def _send_to_vertex(client: MagicMock, caller_headers: Mapping[str, str], drop_params: bool) -> None:
|
||||
with responses.RequestsMock(assert_all_requests_are_fired=False) as oauth:
|
||||
oauth.add(responses.POST, "https://oauth2.googleapis.com/token", json={"access_token": "vertex-token", "expires_in": 3600})
|
||||
await litellm.anthropic.messages.acreate(
|
||||
model="vertex_ai/claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=16,
|
||||
vertex_ai_project="version-project",
|
||||
vertex_ai_location="global",
|
||||
vertex_credentials=json.dumps(
|
||||
{
|
||||
"type": "authorized_user",
|
||||
"client_id": "version-header-client",
|
||||
"client_secret": "synthetic-secret",
|
||||
"refresh_token": "synthetic-refresh",
|
||||
}
|
||||
),
|
||||
client=client,
|
||||
drop_params=drop_params,
|
||||
provider_specific_header={
|
||||
"custom_llm_provider": "vertex_ai",
|
||||
"extra_headers": {**caller_headers, "x-request-id": "req-1"},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("caller_headers", "drop_params", "body_version"),
|
||||
(
|
||||
({"anthropic-version": "2026-10-21"}, False, "2026-10-21"),
|
||||
({"Anthropic-Version": "2026-10-21"}, False, "2026-10-21"),
|
||||
({"anthropic-version": "vertex-2023-10-16"}, False, "vertex-2023-10-16"),
|
||||
({"anthropic-version": "2023-06-01"}, False, "vertex-2023-10-16"),
|
||||
({"anthropic-version": "2023-01-01"}, False, "vertex-2023-10-16"),
|
||||
({"anthropic-version": " "}, False, "vertex-2023-10-16"),
|
||||
({}, False, "vertex-2023-10-16"),
|
||||
({"anthropic-version": "latest"}, True, "vertex-2023-10-16"),
|
||||
),
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_caller_anthropic_version_reaches_vertex_only_in_the_body(
|
||||
caller_headers: dict[str, str], drop_params: bool, body_version: str
|
||||
) -> None:
|
||||
client: Final = _vertex_client()
|
||||
await _send_to_vertex(client, caller_headers, drop_params)
|
||||
|
||||
sent: Final = client.post.call_args.kwargs
|
||||
sent_header_names: Final = {name.lower() for name in sent["headers"]}
|
||||
assert "anthropic-version" not in sent_header_names
|
||||
assert sent["headers"]["x-request-id"] == "req-1"
|
||||
assert json.loads(sent["data"])["anthropic_version"] == body_version
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested", ("2025-01-01", "latest"))
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsupported_anthropic_version_is_refused_without_drop_params(requested: str) -> None:
|
||||
client: Final = _vertex_client()
|
||||
with pytest.raises(litellm.BadRequestError, match=f"anthropic-version={requested}. To drop unsupported params"):
|
||||
await _send_to_vertex(client, {"anthropic-version": requested}, drop_params=False)
|
||||
client.post.assert_not_awaited()
|
||||
|
||||
|
||||
def test_validate_environment_uses_vertex_ai_location():
|
||||
config = VertexAIPartnerModelsAnthropicMessagesConfig()
|
||||
headers = {}
|
||||
|
|
|
|||
|
|
@ -170,7 +170,7 @@ async def test_preview_and_serving_share_default_model_resolution(
|
|||
explicit_default: str | None,
|
||||
expected: str,
|
||||
):
|
||||
from litellm.router_utils.auto_router_model_naming import validate_complexity_router_config_write
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import validate_complexity_router_config_write
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import ComplexityRouterConfigValidationRequest
|
||||
|
||||
config: Final = {
|
||||
|
|
@ -1240,7 +1240,7 @@ class TestAutoRouterBenchmarks:
|
|||
def test_the_listed_kinds_match_the_router_types_traffic_can_record(self):
|
||||
from typing import get_args, get_type_hints
|
||||
|
||||
from litellm.router_utils.auto_router_model_naming import StrategyRouterKind
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import StrategyRouterKind
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
|
||||
readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import inspect
|
||||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import json
|
||||
import socket
|
||||
from collections.abc import Iterator, Mapping
|
||||
|
|
@ -1461,10 +1461,10 @@ class TestDeleteModelClearsRouterRegistry:
|
|||
add_deployment never restores, so an unguarded pop would make it permanently
|
||||
unroutable (the same cross-tenant DoS clear_cache was hardened against).
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import ModelInfoDelete
|
||||
|
||||
model_id = "regular-del-1"
|
||||
admin_user = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
|
|
@ -2739,11 +2739,11 @@ class TestAddAndDeleteModelLifecycle:
|
|||
- Delete same model again → raises (model not found)
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
add_new_model,
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
model_id = "lifecycle-test-model-123"
|
||||
|
|
@ -2833,6 +2833,8 @@ class TestDeleteTeamBYOKModelGhost:
|
|||
async def test_delete_strips_public_name_and_refreshes_cache(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -2916,6 +2918,8 @@ class TestDeleteTeamBYOKModelGhost:
|
|||
alias cleanup (delete_team_model_alias), preserving legacy behavior."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -2984,6 +2988,8 @@ class TestDeleteTeamBYOKModelGhost:
|
|||
team.models when one replica is deleted but a sibling still backs it."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -3065,6 +3071,8 @@ class TestDeleteTeamBYOKModelGhost:
|
|||
deployment still serves it."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -3145,6 +3153,8 @@ class TestDeleteTeamBYOKModelGhost:
|
|||
the alias would break routing that works."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -3252,6 +3262,8 @@ class TestDeleteModelTeamAuth:
|
|||
async def test_proxy_admin_can_delete_model_when_team_deleted(self):
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
@ -3287,6 +3299,8 @@ class TestDeleteModelTeamAuth:
|
|||
"""A missing team must never let a non-admin delete the orphan (no fail-open)."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ProxyException
|
||||
|
|
@ -3322,6 +3336,8 @@ class TestDeleteModelTeamAuth:
|
|||
"""The auth check must not add a redundant team query on the live-team path."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ProxyException
|
||||
|
|
@ -4392,7 +4408,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_echoed_cost_map_metadata_is_not_persisted(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4418,7 +4433,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_an_edited_value_survives_the_echo_filter(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4483,7 +4497,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_metadata_without_a_cost_map_key_is_persisted(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
from litellm.types.utils import echoed_cost_map_fields
|
||||
|
|
@ -4513,7 +4526,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_a_stored_mode_survives_an_echoed_save(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4536,7 +4548,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_resetting_an_override_to_the_cost_map_value_removes_it(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4560,7 +4571,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_reset_is_recognised_after_the_router_registered_the_override(self, monkeypatch: pytest.MonkeyPatch):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4586,7 +4596,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
from types import MappingProxyType
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4611,7 +4620,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_echo_is_compared_against_the_deployments_lookup_not_the_key(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4653,7 +4661,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_base_model_wins_over_litellm_params_model_for_the_lookup(self):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -4681,7 +4688,6 @@ class TestModelInfoCostMapEchoFilter:
|
|||
|
||||
def test_encrypted_stored_model_is_decrypted_for_the_lookup(self, monkeypatch):
|
||||
import litellm
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
|
||||
|
||||
|
|
@ -6431,7 +6437,7 @@ class TestStrategyRouterWriteValidation:
|
|||
AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,
|
||||
_auto_router_capability_slot,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import gated_capabilities_of
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import gated_capabilities_of
|
||||
|
||||
capabilities: Final = gated_capabilities_of(effective_params)
|
||||
|
||||
|
|
@ -6568,7 +6574,7 @@ class TestStrategyRouterWriteValidation:
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
|
||||
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
|
||||
configs = {
|
||||
"_TUNED_A": self._TUNED_A,
|
||||
|
|
@ -6624,7 +6630,7 @@ class TestStrategyRouterWriteValidation:
|
|||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
|
||||
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
|
||||
fake = self._FakeDb([])
|
||||
fake.tx_obj.litellm_proxymodeltable.find_many = AsyncMock(
|
||||
|
|
@ -6669,7 +6675,7 @@ class TestStrategyRouterWriteValidation:
|
|||
row id before the slot is entered, so a second tuned router is refused before its DB write."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import add_new_model
|
||||
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
|
||||
admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
|
||||
baselines = snapshot_tuning_baselines([self._db_router_row("a", self._TUNED_A)])
|
||||
|
|
@ -8980,7 +8986,10 @@ class TestTeamMemberAutoRouterWrites:
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
|
||||
from litellm.proxy.management_helpers.auto_router_permissions import MemberAutoRouterWrite, validate_member_auto_router_config
|
||||
from litellm.proxy.management_helpers.auto_router_permissions import (
|
||||
MemberAutoRouterWrite,
|
||||
validate_member_auto_router_config,
|
||||
)
|
||||
|
||||
row: Final = self._row()
|
||||
database: Final = self._database(self._team(), row)
|
||||
|
|
@ -10677,6 +10686,8 @@ class TestFederationGateScopesToWhatTheWriteTouches:
|
|||
and the team that owns the model must be able to take it off their page."""
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
ModelInfoDelete,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
delete_model as delete_model_endpoint,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,15 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from types import SimpleNamespace
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
|
||||
from litellm.proxy.management_helpers.auto_router_availability import (
|
||||
auto_router_availability,
|
||||
build_auto_router_catalog,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import snapshot_tuning_baselines
|
||||
|
||||
|
||||
def deployment(
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ from pydantic import BaseModel
|
|||
from typing_extensions import TypedDict
|
||||
|
||||
import litellm.proxy.proxy_server as ps
|
||||
from litellm.proxy.management_helpers.audit_logs import track_audit_task
|
||||
from litellm.proxy.proxy_server import (
|
||||
ProxyStartupEvent,
|
||||
initialize_shared_aiohttp_session,
|
||||
_resolve_pydantic_type,
|
||||
_resolve_typed_dict_type,
|
||||
cleanup_router_config_variables,
|
||||
|
|
@ -43,6 +43,7 @@ from litellm.proxy.proxy_server import (
|
|||
get_litellm_model_info,
|
||||
initialize,
|
||||
initialize_from_worker_config,
|
||||
initialize_shared_aiohttp_session,
|
||||
load_from_azure_key_vault,
|
||||
proxy_shutdown_event,
|
||||
proxy_startup_event,
|
||||
|
|
@ -50,8 +51,6 @@ from litellm.proxy.proxy_server import (
|
|||
save_worker_config,
|
||||
)
|
||||
|
||||
from litellm.proxy.management_helpers.audit_logs import track_audit_task
|
||||
|
||||
from .conftest import normalize
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -1165,7 +1164,7 @@ def test_proxy_startup_event_warns_for_global_budget_without_database():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row():
|
||||
from litellm.router_utils.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import DEFAULT_TUNING_FINGERPRINT
|
||||
|
||||
prisma_client = MagicMock()
|
||||
prisma_client.db.litellm_config.find_unique = AsyncMock(return_value=None)
|
||||
|
|
@ -1186,7 +1185,10 @@ async def test_tuning_baseline_v3_is_created_alongside_the_legacy_row():
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scorer_baseline_upgrade_preserves_existing_routers_and_is_not_refreshed_on_restart():
|
||||
from litellm.router_utils.auto_router_tuning_baseline import mutable_tuned_identities, snapshot_tuning_baselines
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
mutable_tuned_identities,
|
||||
snapshot_tuning_baselines,
|
||||
)
|
||||
|
||||
deployments = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -3,9 +3,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
carries_complexity_router_settings,
|
||||
|
|
@ -18,6 +16,8 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
validate_complexity_router_config_write,
|
||||
validate_strategy_router_model_write,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
from litellm.router_strategy.complexity_router.jev_classifier import DEFAULT_JEV_INSTRUCTIONS
|
||||
|
||||
COMPLEXITY_FIELDS = frozenset({"complexity_router_config"})
|
||||
SEMANTIC_FIELDS = frozenset({"auto_router_config", "auto_router_default_model", "auto_router_embedding_model"})
|
||||
|
|
@ -596,7 +596,7 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec
|
|||
],
|
||||
)
|
||||
def test_is_complexity_router_model(model: str | None, expected: bool) -> None:
|
||||
from litellm.router_utils.auto_router_model_naming import is_complexity_router_model
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import is_complexity_router_model
|
||||
|
||||
assert is_complexity_router_model(model) is expected
|
||||
|
||||
|
|
@ -7,8 +7,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
||||
DEFAULT_TUNING_FINGERPRINT,
|
||||
HEURISTIC_V1_TUNING_FIELDS,
|
||||
heuristic_v1_router_fingerprint,
|
||||
|
|
@ -19,6 +18,7 @@ from litellm.router_utils.auto_router_tuning_baseline import (
|
|||
tuning_limit_violation,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
|
||||
_TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}
|
||||
_ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.router_utils.baseline_request import baseline_request, capture_baseline_parameters
|
||||
from litellm.router_strategy.auto_router.baseline_request import baseline_request, capture_baseline_parameters
|
||||
|
||||
|
||||
def test_baseline_snapshot_owns_nested_caller_settings_and_overrides_routed_settings() -> None:
|
||||
|
|
@ -24,14 +24,6 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
CUSTOMIZATION_CAPABILITY,
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
HEURISTIC_V2_CAPABILITY,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
|
|
@ -42,7 +34,19 @@ from litellm.constants import (
|
|||
RETURN_RAW_MODEL_NAME_METADATA_KEY,
|
||||
SESSION_ID_GENERATED_METADATA_KEY,
|
||||
)
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
from litellm.router import as_output_cap, stamped_client_ceiling
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import (
|
||||
CUSTOMIZATION_CAPABILITY,
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
HEURISTIC_V2_CAPABILITY,
|
||||
count_capability_routers,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.capability_classifier import (
|
||||
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT,
|
||||
CapabilityClassifierVerdict,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.complexity_router import (
|
||||
_CLASSIFICATION_CURRENT_MESSAGE_ONLY,
|
||||
_CLASSIFICATION_WITH_CONVERSATION,
|
||||
|
|
@ -58,18 +62,14 @@ from litellm.router_strategy.complexity_router.complexity_router import (
|
|||
classification_system_prompt,
|
||||
custom_tier_classification_prompt,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.capability_classifier import (
|
||||
CAPABILITY_CLASSIFIER_SYSTEM_PROMPT,
|
||||
CapabilityClassifierVerdict,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.config import (
|
||||
CapabilityCalibrationConfig,
|
||||
CapabilityClassifierConfig,
|
||||
DEFAULT_CLASSIFICATION_RUBRIC,
|
||||
DEFAULT_CLASSIFIER_CONTEXT_WINDOW_SIZE,
|
||||
DEFAULT_COMPLEXITY_CONFIG,
|
||||
DEFAULT_TECHNICAL_KEYWORDS,
|
||||
TIER_SEVERITY_ORDER,
|
||||
CapabilityCalibrationConfig,
|
||||
CapabilityClassifierConfig,
|
||||
ClassificationRubric,
|
||||
ClassifierLLMConfig,
|
||||
ComplexityRouterConfig,
|
||||
|
|
@ -83,10 +83,12 @@ from litellm.router_strategy.complexity_router.jev_classifier import (
|
|||
JevUsage,
|
||||
)
|
||||
from litellm.router_strategy.complexity_router.llm_v2 import LLM_V2_PROMPT_VERSION
|
||||
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
|
||||
from litellm.router_strategy.complexity_router.tier_predictor import (
|
||||
TierGlobalStatistic,
|
||||
TrainedTierArtifact,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.router import (
|
||||
Deployment,
|
||||
LiteLLM_Params,
|
||||
|
|
@ -94,10 +96,7 @@ from litellm.types.router import (
|
|||
RouterErrors,
|
||||
TaggedPreRoutingStrategy,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.router_strategy.complexity_router.request_models import RequestComplexityRouterConfig
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
from litellm.types.utils import StandardLoggingRoutingDecision
|
||||
|
||||
requires_semantic_router = pytest.mark.skipif(
|
||||
sys.version_info >= (3, 14), reason="The semantic-router extra excludes Python 3.14"
|
||||
|
|
@ -16299,7 +16298,6 @@ class TestTierHealthFailover:
|
|||
tests can prove the probe passes a kwargs copy and forwards the prompt arguments.
|
||||
"""
|
||||
import litellm as litellm_module
|
||||
|
||||
from litellm.types.router import RouterRateLimitError
|
||||
|
||||
probed_kwargs = []
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ from typing import Final
|
|||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
import litellm
|
||||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse, Router
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, ComplexityTier
|
||||
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
|
||||
|
|
@ -21,7 +22,6 @@ from litellm.router_strategy.complexity_router.llm_v2 import (
|
|||
LLMV2Verdict,
|
||||
llm_v2_response_format,
|
||||
)
|
||||
from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1969,7 +1969,7 @@ ALIAS_CASES: Final = (
|
|||
("litellm.router_utils.add_retry_fallback_headers", "HiddenParamsAsyncIteratorWrapper", "_inner", "inner", True),
|
||||
("litellm.router_utils.add_retry_fallback_headers", "", "_HiddenParamsHost", "HiddenParamsHost", False),
|
||||
(
|
||||
"litellm.router_utils.batch_utils",
|
||||
"litellm.litellm_core_utils.batch_utils",
|
||||
"",
|
||||
"_get_router_metadata_variable_name",
|
||||
"get_router_metadata_variable_name",
|
||||
|
|
|
|||
|
|
@ -4,20 +4,21 @@ import gc
|
|||
import json
|
||||
import os
|
||||
import weakref
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Iterator, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from types import MethodType
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook
|
||||
from litellm.types.litellm_params import RoutingStrategyName
|
||||
|
|
@ -801,10 +802,11 @@ async def test_deployment_callback_on_failure(model_list):
|
|||
|
||||
def test_deployment_callback_respects_cooldown_time(model_list):
|
||||
"""Ensure per-model cooldown_time is honored even when exception headers are present."""
|
||||
import httpx
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import httpx
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
|
||||
class FakeException(Exception):
|
||||
|
|
@ -1811,8 +1813,8 @@ def test_get_allowed_fails_from_policy(model_list, exception_type, exception_nam
|
|||
|
||||
def test_initialize_alerting(model_list):
|
||||
"""Test if the 'initialize_alerting' function is working correctly"""
|
||||
from litellm.types.router import AlertingConfig
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.types.router import AlertingConfig
|
||||
|
||||
router = Router(
|
||||
model_list=model_list, alerting_config=AlertingConfig(webhook_url="test")
|
||||
|
|
@ -2245,6 +2247,7 @@ def test_deployments_by_pattern(model_list):
|
|||
)
|
||||
def test_pattern_match_deployment_set_model_name(user_request_model, model_name, litellm_model, expected_model):
|
||||
from re import Match
|
||||
|
||||
from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
|
||||
|
||||
pattern_router = PatternMatchRouter()
|
||||
|
|
@ -2702,7 +2705,7 @@ def test_handle_clientside_credential_metadata_loading(model_list, function_name
|
|||
)
|
||||
def test_handle_clientside_credential_metadata_variable_name(model_list, function_name, metadata_key):
|
||||
"""Test that _handle_clientside_credential uses the correct metadata variable name based on function name"""
|
||||
from litellm.router_utils.batch_utils import get_router_metadata_variable_name
|
||||
from litellm.litellm_core_utils.batch_utils import get_router_metadata_variable_name
|
||||
|
||||
router = Router(model_list=model_list)
|
||||
|
||||
|
|
|
|||
58
tests/unit/test_router/test_router_surface_check.py
Normal file
58
tests/unit/test_router/test_router_surface_check.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
"""
|
||||
The router surface fence (tests/code_coverage_tests/router_surface_check.py) must see every way
|
||||
outside code reaches a Router member, or new dependencies slip past the shrink-only allowlist.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import importlib.util
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
_CHECK = pathlib.Path(__file__).parents[2] / "code_coverage_tests" / "router_surface_check.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def surface_check():
|
||||
spec = importlib.util.spec_from_file_location("router_surface_check", _CHECK)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _uses(surface_check, src: str) -> set:
|
||||
return surface_check.member_uses(ast.parse(src), surface_check.router_members())
|
||||
|
||||
|
||||
def test_instance_and_dynamic_access_are_uses(surface_check):
|
||||
assert _uses(surface_check, "llm_router.get_model_list()\ngetattr(router, 'model_list')") == {
|
||||
"get_model_list",
|
||||
"model_list",
|
||||
}
|
||||
|
||||
|
||||
def test_class_level_calls_are_uses(surface_check):
|
||||
assert _uses(surface_check, "litellm.Router.get_valid_args()") == {"get_valid_args"}
|
||||
assert _uses(surface_check, "from litellm import Router\nRouter.get_valid_args()") == {"get_valid_args"}
|
||||
|
||||
|
||||
def test_container_file_endpoints_registered_by_name_are_members(surface_check):
|
||||
members = surface_check.router_members()
|
||||
assert {"aupload_container_file", "upload_container_file", "alist_container_files"} <= members
|
||||
assert _uses(surface_check, "await llm_router.aupload_container_file(file=f)") == {"aupload_container_file"}
|
||||
|
||||
|
||||
def test_unlisted_class_level_use_fails_the_fence(surface_check):
|
||||
found = {
|
||||
"members": {"outside": {"get_valid_args": {"litellm/new_module.py"}}, "router_family": {}},
|
||||
"router_utils_imports": {},
|
||||
}
|
||||
allow = {"members_outside": [], "members_router_family": [], "router_utils_imports_outside": []}
|
||||
errors = surface_check.compare(allow, found)
|
||||
assert len(errors) == 1
|
||||
assert "'get_valid_args' in litellm/new_module.py" in errors[0]
|
||||
|
||||
|
||||
def test_unrelated_receivers_are_not_uses(surface_check):
|
||||
assert _uses(surface_check, "client.get_model_list()\nRouterConfig.model_list") == set()
|
||||
Loading…
Add table
Reference in a new issue