mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 18585d663e into c9f7896ed1
This commit is contained in:
commit
b42833cf92
28 changed files with 6638 additions and 14 deletions
19
litellm-rust/Cargo.lock
generated
19
litellm-rust/Cargo.lock
generated
|
|
@ -4331,6 +4331,7 @@ dependencies = [
|
|||
name = "litellm-python-bridge"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
|
|
@ -4357,6 +4358,7 @@ dependencies = [
|
|||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-model-catalog",
|
||||
"litellm-router-core",
|
||||
"litellm-router-types",
|
||||
"litellm-secret-value",
|
||||
"litellm-secrets",
|
||||
|
|
@ -4439,6 +4441,23 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "litellm-router-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"litellm-cache",
|
||||
"litellm-router-types",
|
||||
"parking_lot",
|
||||
"rand 0.8.7",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"sha2 0.10.9",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-router-types"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ litellm-config = { path = "crates/config" }
|
|||
litellm-error = { path = "crates/error" }
|
||||
litellm-router = { path = "crates/router" }
|
||||
litellm-router-types = { path = "crates/router-types" }
|
||||
litellm-router-core = { path = "crates/router-core" }
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
litellm-spend-clickhouse = { path = "crates/spend-clickhouse" }
|
||||
litellm-storage-clickhouse = { path = "crates/storage-clickhouse" }
|
||||
|
|
@ -78,6 +79,8 @@ litellm-cost = { path = "crates/cost" }
|
|||
litellm-model-catalog = { path = "crates/model-catalog" }
|
||||
litellm-model-catalog-refresh = { path = "crates/model-catalog-refresh" }
|
||||
|
||||
arc-swap = "1.9.2"
|
||||
async-trait = "0.1.91"
|
||||
askama = { version = "0.16.1", default-features = false, features = ["derive", "std"] }
|
||||
tracing = "0.1"
|
||||
axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "multipart"] }
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ litellm-tracing.workspace = true
|
|||
litellm-spend-clickhouse.workspace = true
|
||||
litellm-storage-clickhouse.workspace = true
|
||||
litellm-host.workspace = true
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
|
|
@ -49,6 +50,7 @@ litellm-secrets = { workspace = true, features = ["aws", "azure", "google", "has
|
|||
litellm-secrets-types.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-router-core.workspace = true
|
||||
litellm-host-python.workspace = true
|
||||
litellm-model-catalog.workspace = true
|
||||
litellm-token-counter = { path = "../token-counter", default-features = false }
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ mod marshal;
|
|||
mod model_registry;
|
||||
mod preflight;
|
||||
mod python_settings;
|
||||
mod router;
|
||||
mod routes;
|
||||
mod secrets;
|
||||
mod tokenizer;
|
||||
|
|
@ -36,6 +37,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::model_registry::bundled_model_cost_map;
|
||||
#[pymodule_export]
|
||||
use crate::router::{RouterCall, RouterCoreHandle};
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{acompletion, completion};
|
||||
|
|
@ -120,6 +123,8 @@ mod tests {
|
|||
"bundled_model_cost_map",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"RouterCoreHandle",
|
||||
"RouterCall",
|
||||
"gil_stats",
|
||||
"process_state_started",
|
||||
"reserve_process_for_forking",
|
||||
|
|
|
|||
294
litellm-rust/crates/python-bridge/src/router/config.rs
Normal file
294
litellm-rust/crates/python-bridge/src/router/config.rs
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
//! The documents `RouterCoreHandle` reads: the router's static and dynamic config, a call's
|
||||
//! request and an `apply` change set. A config key router-core does not read declines the
|
||||
//! handle, so Python keeps the router on its own core.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use litellm_router_core::{
|
||||
ChangeSet, ConfigError, DeploymentSpec, RouterConfig, RouterDynamicConfig, RouterRequest,
|
||||
};
|
||||
use serde::{Deserialize, de::DeserializeOwned};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
/// The `RouterConfig` keys router-core reads.
|
||||
const STATIC_KEYS: &[&str] = &[
|
||||
"model_list",
|
||||
"routing_strategy",
|
||||
"routing_groups",
|
||||
"plugins",
|
||||
"cooldown_time",
|
||||
"disable_cooldowns",
|
||||
"allowed_fails_policy",
|
||||
"retry_policy",
|
||||
"retry_after",
|
||||
"timeout",
|
||||
"request_timeout",
|
||||
"drop_params",
|
||||
"enable_pre_call_checks",
|
||||
"enable_health_check_routing",
|
||||
"provider_default_deployment_ids",
|
||||
];
|
||||
|
||||
/// The `RouterDynamicConfig` keys router-core reads.
|
||||
const DYNAMIC_KEYS: &[&str] = &[
|
||||
"version",
|
||||
"model_alias_map",
|
||||
"model_group_alias",
|
||||
"fallbacks",
|
||||
"default_fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"num_retries",
|
||||
"max_fallbacks",
|
||||
"allowed_fails",
|
||||
"fallback_keys",
|
||||
];
|
||||
|
||||
/// Router arguments that do not change routing, accepted in either config and dropped.
|
||||
const IGNORED_KEYS: &[&str] = &[
|
||||
"set_verbose",
|
||||
"debug_level",
|
||||
"alerting_config",
|
||||
"client_ttl",
|
||||
];
|
||||
|
||||
/// Fallback lists Python treats as unset when empty (`context_window_fallbacks or None`).
|
||||
const EMPTY_MEANS_UNSET: &[&str] = &["context_window_fallbacks", "content_policy_fallbacks"];
|
||||
|
||||
/// Which document a failure is about.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)]
|
||||
pub(crate) enum Document {
|
||||
#[strum(serialize = "router config")]
|
||||
Static,
|
||||
#[strum(serialize = "dynamic router config")]
|
||||
Dynamic,
|
||||
#[strum(serialize = "router request")]
|
||||
Request,
|
||||
#[strum(serialize = "change set")]
|
||||
Changes,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum Error {
|
||||
#[error("the {document} must be a JSON object")]
|
||||
NotObject { document: Document },
|
||||
#[error("router-core does not read `{key}` in the {document}")]
|
||||
UnknownKey { document: Document, key: String },
|
||||
#[error("router-core does not run `{key}` = {value}")]
|
||||
UnsupportedValue { key: &'static str, value: Value },
|
||||
#[error("the {document} does not have the shape router-core reads")]
|
||||
Shape {
|
||||
document: Document,
|
||||
#[source]
|
||||
source: serde_json::Error,
|
||||
},
|
||||
#[error("the request names no model")]
|
||||
MissingModel,
|
||||
#[error("router-core rejected the router config")]
|
||||
Core(#[from] ConfigError),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// Whether router-core cannot serve this config, so Python keeps its own core.
|
||||
pub(crate) fn declined(&self) -> bool {
|
||||
match self {
|
||||
Self::UnknownKey { .. } | Self::UnsupportedValue { .. } | Self::Core(_) => true,
|
||||
Self::Shape { document, .. } => {
|
||||
matches!(document, Document::Static | Document::Dynamic)
|
||||
}
|
||||
Self::NotObject { .. } | Self::MissingModel => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn object(document: Document, value: Value) -> Result<Map<String, Value>, Error> {
|
||||
match value {
|
||||
Value::Object(object) => Ok(object),
|
||||
_ => Err(Error::NotObject { document }),
|
||||
}
|
||||
}
|
||||
|
||||
fn shaped<T: DeserializeOwned>(document: Document, object: Map<String, Value>) -> Result<T, Error> {
|
||||
serde_json::from_value(Value::Object(object))
|
||||
.map_err(|source| Error::Shape { document, source })
|
||||
}
|
||||
|
||||
/// Drop unset and ignored keys, then reject any key outside `known`.
|
||||
fn routing_keys(
|
||||
document: Document,
|
||||
value: Value,
|
||||
known: &[&str],
|
||||
) -> Result<Map<String, Value>, Error> {
|
||||
let mut object = object(document, value)?;
|
||||
object.retain(|key, value| {
|
||||
let empty_unset = EMPTY_MEANS_UNSET.contains(&key.as_str())
|
||||
&& value.as_array().is_some_and(Vec::is_empty);
|
||||
!(value.is_null() || empty_unset || IGNORED_KEYS.contains(&key.as_str()))
|
||||
});
|
||||
if let Some(key) = object.keys().find(|key| !known.contains(&key.as_str())) {
|
||||
return Err(Error::UnknownKey {
|
||||
document,
|
||||
key: key.clone(),
|
||||
});
|
||||
}
|
||||
Ok(object)
|
||||
}
|
||||
|
||||
pub(crate) fn router_config(value: Value) -> Result<RouterConfig, Error> {
|
||||
let object = routing_keys(Document::Static, value, STATIC_KEYS)?;
|
||||
// Pre-call checks reach router-core as plugins; the Python flag alone has no native filter.
|
||||
if let Some(value) = object.get("enable_pre_call_checks")
|
||||
&& value != &Value::Bool(false)
|
||||
{
|
||||
return Err(Error::UnsupportedValue {
|
||||
key: "enable_pre_call_checks",
|
||||
value: value.clone(),
|
||||
});
|
||||
}
|
||||
shaped(Document::Static, object)
|
||||
}
|
||||
|
||||
pub(crate) fn dynamic_config(value: Value) -> Result<RouterDynamicConfig, Error> {
|
||||
let object = routing_keys(Document::Dynamic, value, DYNAMIC_KEYS)?;
|
||||
shaped(Document::Dynamic, object)
|
||||
}
|
||||
|
||||
/// A call's request as Python spells it.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct RequestSpec {
|
||||
model: Option<String>,
|
||||
team_id: Option<String>,
|
||||
specific_deployment: Option<bool>,
|
||||
num_retries: Option<u32>,
|
||||
disable_fallbacks: Option<bool>,
|
||||
router_weights: Option<BTreeMap<String, BTreeMap<String, f64>>>,
|
||||
body: Option<Value>,
|
||||
}
|
||||
|
||||
/// Read a request; `model` replaces the request's own when given.
|
||||
pub(crate) fn request(value: Value, model: Option<&str>) -> Result<RouterRequest, Error> {
|
||||
let spec: RequestSpec = shaped(Document::Request, object(Document::Request, value)?)?;
|
||||
let model = model
|
||||
.map(str::to_owned)
|
||||
.or(spec.model)
|
||||
.ok_or(Error::MissingModel)?;
|
||||
Ok(RouterRequest {
|
||||
model,
|
||||
body: spec.body.unwrap_or_default(),
|
||||
team_id: spec.team_id,
|
||||
specific_deployment: spec.specific_deployment.unwrap_or(false),
|
||||
num_retries: spec.num_retries,
|
||||
disable_fallbacks: spec.disable_fallbacks.unwrap_or(false),
|
||||
router_weights: spec.router_weights,
|
||||
payload: None,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ChangesSpec {
|
||||
#[serde(default)]
|
||||
upsert: Vec<DeploymentSpec>,
|
||||
#[serde(default)]
|
||||
delete: Vec<String>,
|
||||
}
|
||||
|
||||
pub(crate) fn changes(value: Value) -> Result<ChangeSet, Error> {
|
||||
let spec: ChangesSpec = shaped(Document::Changes, object(Document::Changes, value)?)?;
|
||||
Ok(ChangeSet {
|
||||
upserts: spec.upsert,
|
||||
deletes: spec.delete,
|
||||
settings: None,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown_key(json!({"routing_strategy_args": {"ttl": 1}}), "routing_strategy_args")]
|
||||
#[case::dynamic_key_in_static(json!({"num_retries": 2}), "num_retries")]
|
||||
fn an_unread_static_key_declines_naming_the_key(#[case] json: Value, #[case] key: &str) {
|
||||
let error = router_config(json).unwrap_err();
|
||||
assert!(error.declined());
|
||||
assert!(matches!(&error, Error::UnknownKey { key: named, .. } if named == key));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::null_unknown(json!({"routing_strategy_args": null}))]
|
||||
#[case::ignored(json!({"set_verbose": true, "debug_level": "INFO"}))]
|
||||
#[case::pre_call_checks_off(json!({"enable_pre_call_checks": false}))]
|
||||
fn unset_and_ignored_keys_are_accepted(#[case] json: Value) {
|
||||
router_config(json).unwrap();
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn pre_call_checks_without_a_plugin_decline() {
|
||||
let error = router_config(json!({"enable_pre_call_checks": true})).unwrap_err();
|
||||
assert!(error.declined());
|
||||
assert!(matches!(
|
||||
error,
|
||||
Error::UnsupportedValue {
|
||||
key: "enable_pre_call_checks",
|
||||
..
|
||||
}
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_object(json!([]), false)]
|
||||
#[case::bad_shape(json!({"retry_after": "soon"}), true)]
|
||||
fn a_config_that_is_not_an_object_is_a_value_error_and_a_wrong_shape_declines(
|
||||
#[case] json: Value,
|
||||
#[case] declined: bool,
|
||||
) {
|
||||
assert_eq!(router_config(json).unwrap_err().declined(), declined);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn python_router_defaults_read_as_unset() {
|
||||
let dynamic = dynamic_config(json!({
|
||||
"num_retries": null, "fallbacks": [], "context_window_fallbacks": [],
|
||||
"content_policy_fallbacks": [], "model_group_alias": {"gpt": "gpt-4"},
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(dynamic.num_retries, None);
|
||||
assert!(dynamic.context_window_fallbacks.is_none());
|
||||
assert!(dynamic.content_policy_fallbacks.is_none());
|
||||
assert_eq!(dynamic.model_alias_map["gpt"], "gpt-4");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::own_model(json!({"model": "gpt", "num_retries": 1}), None, "gpt")]
|
||||
#[case::override_model(json!({"model": "gpt"}), Some("other"), "other")]
|
||||
fn a_request_reads_its_model(
|
||||
#[case] json: Value,
|
||||
#[case] model: Option<&str>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(request(json, model).unwrap().model, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown_field(json!({"model": "gpt", "temperature": 1}))]
|
||||
#[case::no_model(json!({"team_id": "t"}))]
|
||||
fn a_bad_request_is_a_value_error(#[case] json: Value) {
|
||||
assert!(!request(json, None).unwrap_err().declined());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_change_set_reads_upserts_and_deletes() {
|
||||
let changes = changes(json!({
|
||||
"upsert": [{"model_name": "m", "litellm_params": {"model": "openai/x"}}],
|
||||
"delete": ["old"],
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(changes.upserts.len(), 1);
|
||||
assert_eq!(changes.deletes, ["old"]);
|
||||
}
|
||||
}
|
||||
1011
litellm-rust/crates/python-bridge/src/router/host.rs
Normal file
1011
litellm-rust/crates/python-bridge/src/router/host.rs
Normal file
File diff suppressed because it is too large
Load diff
549
litellm-rust/crates/python-bridge/src/router/mod.rs
Normal file
549
litellm-rust/crates/python-bridge/src/router/mod.rs
Normal file
|
|
@ -0,0 +1,549 @@
|
|||
//! `RouterCoreHandle`: router-core behind a step-driven handle. Python drives each call through
|
||||
//! [`RouterCall::next`], running every attempt, sleep, fallback check, selection and cache op the
|
||||
//! call loop asks for and handing the outcome back; Rust never awaits Python. `select` is driven
|
||||
//! the same way, since reading cooldowns is a cache step.
|
||||
|
||||
mod config;
|
||||
mod host;
|
||||
|
||||
use std::{
|
||||
sync::{Arc, Mutex, PoisonError},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use litellm_error::Report;
|
||||
use litellm_host_python::{from_py_argument, json_loads, to_py};
|
||||
use litellm_router_core::{AttemptError, Error, FailureKind, Op, Router, RouterEvent};
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
types::{PyBool, PyDict, PyList, PyString},
|
||||
};
|
||||
use serde_json::Value;
|
||||
|
||||
use self::host::{
|
||||
Answer, Ask, CacheOp, CacheOutcome, DriveError, Driver, Finish, Stage, Step, StepHost, StepKind,
|
||||
};
|
||||
use crate::errors::RustBridgeDeclined;
|
||||
|
||||
/// The `error_kind` of a `failed` step.
|
||||
#[derive(Clone, Copy, Debug, strum::IntoStaticStr)]
|
||||
enum ErrorKind {
|
||||
#[strum(serialize = "config")]
|
||||
Config,
|
||||
#[strum(serialize = "unknown_model")]
|
||||
UnknownModel,
|
||||
#[strum(serialize = "no_healthy_deployments")]
|
||||
NoHealthyDeployments,
|
||||
#[strum(serialize = "attempt")]
|
||||
Attempt,
|
||||
#[strum(serialize = "cache")]
|
||||
Cache,
|
||||
#[strum(serialize = "retired")]
|
||||
Retired,
|
||||
}
|
||||
|
||||
impl From<&Error> for ErrorKind {
|
||||
fn from(error: &Error) -> Self {
|
||||
match error {
|
||||
Error::Config(_) => Self::Config,
|
||||
Error::UnknownModel { .. } => Self::UnknownModel,
|
||||
Error::NoHealthyDeployments { .. } => Self::NoHealthyDeployments,
|
||||
Error::Attempt(_) => Self::Attempt,
|
||||
Error::Cache(_) => Self::Cache,
|
||||
Error::Retired => Self::Retired,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn config_error_to_pyerr(error: config::Error) -> PyErr {
|
||||
let message = Report::new(&error).to_string();
|
||||
if error.declined() {
|
||||
RustBridgeDeclined::new_err(message)
|
||||
} else {
|
||||
PyValueError::new_err(message)
|
||||
}
|
||||
}
|
||||
|
||||
fn drive_error_to_pyerr(error: DriveError) -> PyErr {
|
||||
let message = Report::new(&error).to_string();
|
||||
if error.is_request() {
|
||||
PyValueError::new_err(message)
|
||||
} else {
|
||||
PyRuntimeError::new_err(message)
|
||||
}
|
||||
}
|
||||
|
||||
/// A document Python passes as a mapping or as its JSON text.
|
||||
fn document(value: &Bound<'_, PyAny>) -> PyResult<Value> {
|
||||
match value.cast::<PyString>() {
|
||||
Ok(text) => {
|
||||
let parsed = json_loads(value.py(), text.to_str()?.as_bytes())?;
|
||||
from_py_argument(parsed.bind(value.py()))
|
||||
}
|
||||
Err(_) => from_py_argument(value),
|
||||
}
|
||||
}
|
||||
|
||||
/// A router-core router whose host operations Python runs.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct RouterCoreHandle {
|
||||
router: Router,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl RouterCoreHandle {
|
||||
#[new]
|
||||
fn new(config: &Bound<'_, PyAny>, dynamic: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
let (config, dynamic) = (document(config)?, document(dynamic)?);
|
||||
let build = || {
|
||||
let config = config::router_config(config)?;
|
||||
let dynamic = config::dynamic_config(dynamic)?;
|
||||
Ok(Router::new(config, dynamic, Arc::new(StepHost))?)
|
||||
};
|
||||
build()
|
||||
.map(|router| Self { router })
|
||||
.map_err(config_error_to_pyerr)
|
||||
}
|
||||
|
||||
/// Start a call; nothing runs until the first `next(None)`.
|
||||
fn start(&self, op: &str, request: &Bound<'_, PyAny>) -> PyResult<RouterCall> {
|
||||
let op: Op = op.parse().unwrap_or_else(|_| Op::Other(op.to_owned()));
|
||||
let request = config::request(document(request)?, None).map_err(config_error_to_pyerr)?;
|
||||
let router = self.router.clone();
|
||||
Ok(RouterCall::new(async move {
|
||||
router.call(op, request).await.map(Finish::Call)
|
||||
}))
|
||||
}
|
||||
|
||||
/// Pick the deployment `model` routes to now, without calling it. Driven like `start`: its
|
||||
/// steps are the cache reads and selections the pick needs, and it ends with `done`
|
||||
/// (`deployment_id`, `model_group`) or `failed`.
|
||||
fn select(&self, model: &str, request: &Bound<'_, PyAny>) -> PyResult<RouterCall> {
|
||||
let request =
|
||||
config::request(document(request)?, Some(model)).map_err(config_error_to_pyerr)?;
|
||||
let router = self.router.clone();
|
||||
Ok(RouterCall::new(async move {
|
||||
let deployment = router.select(&Op::Completion, &request).await?;
|
||||
Ok(Finish::Selected {
|
||||
deployment_id: deployment.id.clone(),
|
||||
model_group: deployment.model_name.clone(),
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/// Apply `{"upsert": [deployments], "delete": [ids]}` and return the new registry version.
|
||||
fn apply(&self, changes: &Bound<'_, PyAny>) -> PyResult<u64> {
|
||||
let changes = config::changes(document(changes)?).map_err(config_error_to_pyerr)?;
|
||||
self.router
|
||||
.apply(changes)
|
||||
.map_err(|error| config_error_to_pyerr(error.into()))
|
||||
}
|
||||
|
||||
fn registry_version(&self) -> u64 {
|
||||
self.router.registry().version()
|
||||
}
|
||||
|
||||
fn retire(&self) {
|
||||
self.router.retire();
|
||||
}
|
||||
}
|
||||
|
||||
/// One router call, driven a step at a time by Python.
|
||||
#[pyclass(frozen)]
|
||||
pub(crate) struct RouterCall {
|
||||
driver: Mutex<Driver>,
|
||||
}
|
||||
|
||||
impl RouterCall {
|
||||
fn new(
|
||||
future: impl std::future::Future<Output = Result<Finish, Error>> + Send + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
driver: Mutex::new(Driver::new(future)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl RouterCall {
|
||||
/// Hand back the outcome of the last step (`None` on the first call) and return the next
|
||||
/// step, or the call's end.
|
||||
#[pyo3(signature = (outcome))]
|
||||
fn next<'py>(
|
||||
&self,
|
||||
py: Python<'py>,
|
||||
outcome: Option<Bound<'py, PyAny>>,
|
||||
) -> PyResult<Bound<'py, PyDict>> {
|
||||
let mut driver = self.driver.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let answer = match driver.waiting() {
|
||||
None => match outcome {
|
||||
None => None,
|
||||
Some(_) => return Err(drive_error_to_pyerr(DriveError::Unasked)),
|
||||
},
|
||||
Some(ask) => Some(answer(ask, outcome)?),
|
||||
};
|
||||
let step = driver.resume(answer).map_err(drive_error_to_pyerr)?;
|
||||
let dict = step_dict(py, step)?;
|
||||
dict.set_item("events", events_list(py, driver.take_events())?)?;
|
||||
Ok(dict)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read Python's outcome for `ask`.
|
||||
fn answer(ask: &Ask, outcome: Option<Bound<'_, PyAny>>) -> PyResult<Answer> {
|
||||
let missing = || {
|
||||
drive_error_to_pyerr(DriveError::MissingAnswer {
|
||||
expected: ask.kind(),
|
||||
})
|
||||
};
|
||||
match ask {
|
||||
Ask::Sleep { .. } => Ok(Answer::Slept),
|
||||
Ask::AllowFallback { .. } => Ok(Answer::AllowFallback(
|
||||
outcome.ok_or_else(missing)?.extract()?,
|
||||
)),
|
||||
Ask::Select { .. } => Ok(Answer::Select(outcome.ok_or_else(missing)?.extract()?)),
|
||||
Ask::Attempt { .. } => {
|
||||
let outcome = outcome.ok_or_else(missing)?;
|
||||
attempt_outcome(outcome.cast::<PyDict>()?).map(Answer::Attempt)
|
||||
}
|
||||
Ask::Cache { op, .. } => cache_outcome(*op, outcome).map(Answer::Cache),
|
||||
}
|
||||
}
|
||||
|
||||
/// `get`: the JSON value or None; `batch_get`: a list of JSON values or None, one per key;
|
||||
/// `increment`: the new int or float; `set` and `delete`: None.
|
||||
fn cache_outcome(op: CacheOp, outcome: Option<Bound<'_, PyAny>>) -> PyResult<CacheOutcome> {
|
||||
let outcome = outcome.filter(|outcome| !outcome.is_none());
|
||||
let json = |value: Option<Value>| value.filter(|value| !value.is_null());
|
||||
match op {
|
||||
CacheOp::Get => outcome
|
||||
.map(|outcome| from_py_argument::<Value>(&outcome))
|
||||
.transpose()
|
||||
.map(|value| CacheOutcome::Value(json(value))),
|
||||
CacheOp::BatchGet => {
|
||||
let outcome = outcome.ok_or_else(|| {
|
||||
PyValueError::new_err("a `batch_get` cache outcome must be a list, not None")
|
||||
})?;
|
||||
if !outcome.is_instance_of::<PyList>() {
|
||||
return Err(PyValueError::new_err(
|
||||
"a `batch_get` cache outcome must be a list with one entry per key",
|
||||
));
|
||||
}
|
||||
let values: Vec<Option<Value>> = from_py_argument(&outcome)?;
|
||||
Ok(CacheOutcome::Values(values.into_iter().map(json).collect()))
|
||||
}
|
||||
CacheOp::Increment => outcome
|
||||
.filter(|outcome| !outcome.is_instance_of::<PyBool>())
|
||||
.and_then(|outcome| outcome.extract::<f64>().ok())
|
||||
.map(CacheOutcome::Number)
|
||||
.ok_or_else(|| {
|
||||
PyValueError::new_err("an `increment` cache outcome must be the new int or float")
|
||||
}),
|
||||
CacheOp::Set | CacheOp::Delete => match outcome {
|
||||
None => Ok(CacheOutcome::Stored),
|
||||
Some(_) => Err(PyValueError::new_err(format!(
|
||||
"a `{op}` cache step has no outcome, so pass None"
|
||||
))),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// `{"ok": True, "total_tokens": int}` or `{"ok": False, "kind": str, "status": int | None,
|
||||
/// "retry_after": float | None, "num_retries": int | None, "detail": str | None}`.
|
||||
fn attempt_outcome(outcome: &Bound<'_, PyDict>) -> PyResult<Result<u64, AttemptError>> {
|
||||
fn item<'py, T: FromPyObjectOwned<'py>>(
|
||||
outcome: &Bound<'py, PyDict>,
|
||||
key: &str,
|
||||
) -> PyResult<Option<T>> {
|
||||
match outcome.get_item(key)? {
|
||||
Some(value) if !value.is_none() => Ok(Some(value.extract().map_err(Into::into)?)),
|
||||
Some(_) | None => Ok(None),
|
||||
}
|
||||
}
|
||||
let ok: bool = item(outcome, "ok")?
|
||||
.ok_or_else(|| PyValueError::new_err("an attempt outcome needs `ok`"))?;
|
||||
if ok {
|
||||
return Ok(Ok(item(outcome, "total_tokens")?.unwrap_or(0)));
|
||||
}
|
||||
let kind: String = item(outcome, "kind")?
|
||||
.ok_or_else(|| PyValueError::new_err("a failed attempt outcome needs `kind`"))?;
|
||||
let kind: FailureKind = kind
|
||||
.parse()
|
||||
.map_err(|_| PyValueError::new_err(format!("`{kind}` is not a router failure kind")))?;
|
||||
let retry_after = item::<f64>(outcome, "retry_after")?
|
||||
.map(|seconds| {
|
||||
Duration::try_from_secs_f64(seconds)
|
||||
.map_err(|_| PyValueError::new_err("`retry_after` must be a non-negative number"))
|
||||
})
|
||||
.transpose()?;
|
||||
let mut failure = AttemptError::new(kind, item(outcome, "status")?);
|
||||
failure.retry_after = retry_after;
|
||||
failure.num_retries = item(outcome, "num_retries")?;
|
||||
failure.detail = item::<String>(outcome, "detail")?.map(Into::into);
|
||||
Ok(Err(failure))
|
||||
}
|
||||
|
||||
/// `[{"kind": "retries_exhausted", "model_group": str, "attempted": int, "max": int}
|
||||
/// | {"kind": "fallback", "to": str, "order": int | None, "level": int}]`.
|
||||
fn events_list(py: Python<'_>, events: Vec<RouterEvent>) -> PyResult<Bound<'_, PyList>> {
|
||||
let list = PyList::empty(py);
|
||||
for event in events {
|
||||
let dict = PyDict::new(py);
|
||||
match event {
|
||||
RouterEvent::RetriesExhausted {
|
||||
model_group,
|
||||
attempted,
|
||||
max,
|
||||
} => {
|
||||
dict.set_item("kind", "retries_exhausted")?;
|
||||
dict.set_item("model_group", model_group)?;
|
||||
dict.set_item("attempted", attempted)?;
|
||||
dict.set_item("max", max)?;
|
||||
}
|
||||
RouterEvent::Fallback {
|
||||
to, order, level, ..
|
||||
} => {
|
||||
dict.set_item("kind", "fallback")?;
|
||||
dict.set_item("to", to)?;
|
||||
dict.set_item("order", order)?;
|
||||
dict.set_item("level", level)?;
|
||||
}
|
||||
_ => continue,
|
||||
}
|
||||
list.append(dict)?;
|
||||
}
|
||||
Ok(list)
|
||||
}
|
||||
|
||||
fn step_dict(py: Python<'_>, step: Step) -> PyResult<Bound<'_, PyDict>> {
|
||||
let dict = PyDict::new(py);
|
||||
let kind = match &step {
|
||||
Step::Ask(ask) => ask.kind(),
|
||||
Step::Done(_) => StepKind::Done,
|
||||
Step::Failed(_) => StepKind::Failed,
|
||||
};
|
||||
dict.set_item("kind", <&'static str>::from(kind))?;
|
||||
match step {
|
||||
Step::Ask(ask) => ask_items(py, &dict, ask)?,
|
||||
Step::Done(finish) => done_items(&dict, finish)?,
|
||||
Step::Failed(error) => {
|
||||
let attempt = error.attempt();
|
||||
dict.set_item("error", Report::new(&error).to_string())?;
|
||||
dict.set_item("error_kind", <&'static str>::from(ErrorKind::from(&error)))?;
|
||||
dict.set_item("attempt_failed", attempt.is_some())?;
|
||||
dict.set_item("status", attempt.and_then(|failure| failure.status))?;
|
||||
dict.set_item(
|
||||
"failure_kind",
|
||||
attempt.map(|failure| <&'static str>::from(failure.kind)),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
Ok(dict)
|
||||
}
|
||||
|
||||
fn ask_items(py: Python<'_>, dict: &Bound<'_, PyDict>, ask: Ask) -> PyResult<()> {
|
||||
match ask {
|
||||
Ask::Attempt {
|
||||
deployment_id,
|
||||
model,
|
||||
model_group,
|
||||
target_order,
|
||||
candidate_ids,
|
||||
retry,
|
||||
fallback_depth,
|
||||
fallback_level,
|
||||
timeout,
|
||||
} => {
|
||||
dict.set_item("deployment_id", deployment_id)?;
|
||||
dict.set_item("model", model)?;
|
||||
dict.set_item("model_group", model_group)?;
|
||||
dict.set_item("target_order", target_order)?;
|
||||
dict.set_item("candidate_ids", PyList::new(py, candidate_ids)?)?;
|
||||
dict.set_item("retry", retry)?;
|
||||
dict.set_item("fallback_depth", fallback_depth)?;
|
||||
dict.set_item("fallback_level", fallback_level)?;
|
||||
dict.set_item("timeout", timeout)
|
||||
}
|
||||
Ask::Sleep { seconds } => dict.set_item("seconds", seconds),
|
||||
Ask::AllowFallback { target } => dict.set_item("target", target),
|
||||
Ask::Select {
|
||||
stage,
|
||||
model_group,
|
||||
candidates,
|
||||
} => {
|
||||
let (stage, strategy) = match stage {
|
||||
Stage::Plugins => ("plugins", None),
|
||||
Stage::Strategy(strategy) => ("strategy", Some(strategy)),
|
||||
};
|
||||
dict.set_item("stage", stage)?;
|
||||
dict.set_item("strategy", strategy)?;
|
||||
dict.set_item("model_group", model_group)?;
|
||||
dict.set_item("candidates", PyList::new(py, candidates)?)
|
||||
}
|
||||
Ask::Cache {
|
||||
op,
|
||||
key,
|
||||
keys,
|
||||
value,
|
||||
ttl,
|
||||
} => {
|
||||
dict.set_item("op", <&'static str>::from(op))?;
|
||||
dict.set_item("key", key)?;
|
||||
dict.set_item("keys", keys.map(|keys| PyList::new(py, keys)).transpose()?)?;
|
||||
dict.set_item("value", value.map(|value| to_py(py, &value)).transpose()?)?;
|
||||
dict.set_item("ttl", ttl)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn done_items(dict: &Bound<'_, PyDict>, finish: Finish) -> PyResult<()> {
|
||||
match finish {
|
||||
Finish::Call(response) => {
|
||||
dict.set_item("deployment_id", response.deployment_id)?;
|
||||
dict.set_item("model_group", response.model_group)?;
|
||||
dict.set_item("attempted_retries", response.attempted_retries)?;
|
||||
dict.set_item("max_retries", response.max_retries)?;
|
||||
dict.set_item("attempted_fallbacks", response.attempted_fallbacks)?;
|
||||
dict.set_item("total_tokens", response.output.total_tokens)
|
||||
}
|
||||
Finish::Selected {
|
||||
deployment_id,
|
||||
model_group,
|
||||
} => {
|
||||
dict.set_item("deployment_id", deployment_id)?;
|
||||
dict.set_item("model_group", model_group)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use pyo3::{ffi::c_str, prelude::*, types::PyDict};
|
||||
use rstest::rstest;
|
||||
|
||||
const CONFIG: &str = r#"{"model_list": [
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}}
|
||||
]}"#;
|
||||
|
||||
const PAIR: &str = r#"{"model_list": [
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}},
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}}
|
||||
]}"#;
|
||||
|
||||
#[rstest]
|
||||
fn python_drives_calls_and_selections_through_the_handle() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals.set_item("native", crate::native_module(py)).unwrap();
|
||||
locals.set_item("config", CONFIG).unwrap();
|
||||
locals.set_item("pair", PAIR).unwrap();
|
||||
py.run(
|
||||
c_str!(
|
||||
r#"
|
||||
import time
|
||||
store = {}
|
||||
asked = []
|
||||
|
||||
def answer_cache(step):
|
||||
asked.append(step)
|
||||
op, key = step["op"], step["key"]
|
||||
if op == "get":
|
||||
return store.get(key)
|
||||
if op == "batch_get":
|
||||
return [store.get(k) for k in step["keys"]]
|
||||
if op == "set":
|
||||
store[key] = step["value"]
|
||||
return None
|
||||
if op == "increment":
|
||||
store[key] = store.get(key, 0) + step["value"]
|
||||
return store[key]
|
||||
if op == "delete":
|
||||
store.pop(key, None)
|
||||
return None
|
||||
raise AssertionError(op)
|
||||
|
||||
def drive(call, outcome):
|
||||
step = call.next(outcome)
|
||||
while step["kind"] == "cache":
|
||||
step = call.next(answer_cache(step))
|
||||
return step
|
||||
|
||||
handle = native.RouterCoreHandle(config, {"num_retries": 1})
|
||||
picked = drive(handle.select("primary", {}), None)
|
||||
assert (picked["kind"], picked["deployment_id"], picked["model_group"]) == ("done", "a1", "primary"), picked
|
||||
assert [step["op"] for step in asked] == ["batch_get"]
|
||||
assert list(asked[0]) == ["kind", "op", "key", "keys", "value", "ttl", "events"], list(asked[0])
|
||||
call = handle.start("acompletion", '{"model": "primary"}')
|
||||
version = handle.registry_version()
|
||||
assert handle.apply({"delete": ["gone"]}) == version + 1
|
||||
steps = [drive(call, None)]
|
||||
steps.append(drive(call, {"ok": False, "kind": "RateLimitError", "status": 429}))
|
||||
steps.append(drive(call, None))
|
||||
steps.append(drive(call, {"ok": True, "total_tokens": 5}))
|
||||
kinds = [step["kind"] for step in steps]
|
||||
|
||||
cooled = native.RouterCoreHandle(pair, {"num_retries": 0})
|
||||
call = cooled.start("acompletion", {"model": "primary"})
|
||||
tried = drive(call, None)["deployment_id"]
|
||||
assert drive(call, {"ok": False, "kind": "RateLimitError", "status": 429})["kind"] == "failed"
|
||||
assert f"deployment:{tried}:cooldown" not in store, store
|
||||
store[f"deployment:{tried}:cooldown"] = {"exception_received": "x", "status_code": "429", "timestamp": time.time(), "cooldown_time": 60.0}
|
||||
other = drive(cooled.select("primary", {}), None)
|
||||
assert other["kind"] == "done" and other["deployment_id"] != tried, other
|
||||
|
||||
call = handle.select("primary", {})
|
||||
step = call.next(None)
|
||||
assert step["op"] == "batch_get"
|
||||
for bad in ([], [None, None], "nope", None):
|
||||
try:
|
||||
call.next(bad)
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError(f"{bad!r} should not answer a batch_get of one key")
|
||||
assert drive(call, [None])["kind"] == "done"
|
||||
"#
|
||||
),
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap();
|
||||
let kinds: Vec<String> = locals
|
||||
.get_item("kinds")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.extract()
|
||||
.unwrap();
|
||||
assert_eq!(kinds, ["attempt", "sleep", "attempt", "done"]);
|
||||
});
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unknown_key(r#"{"routing_strategy_args": {}}"#, true)]
|
||||
#[case::bad_json("{", false)]
|
||||
#[case::not_an_object("[]", false)]
|
||||
fn construction_failures_raise_the_class_python_reads(
|
||||
#[case] config: &str,
|
||||
#[case] declined: bool,
|
||||
) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let error = crate::native_module(py)
|
||||
.getattr("RouterCoreHandle")
|
||||
.and_then(|handle| handle.call1((config, "{}")))
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error.is_instance_of::<crate::errors::RustBridgeDeclined>(py),
|
||||
declined,
|
||||
"{error}"
|
||||
);
|
||||
assert_eq!(
|
||||
error.is_instance_of::<pyo3::exceptions::PyValueError>(py),
|
||||
!declined,
|
||||
"{error}"
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -7,3 +7,20 @@ repository.workspace = true
|
|||
description = "The router core behind litellm.Router: registry, settings, health, selection and the call loop"
|
||||
|
||||
[dependencies]
|
||||
arc-swap.workspace = true
|
||||
async-trait.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
parking_lot.workspace = true
|
||||
rand.workspace = true
|
||||
serde.workspace = true
|
||||
serde_with.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
sha2.workspace = true
|
||||
strum.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
130
litellm-rust/crates/router-core/src/cache.rs
Normal file
130
litellm-rust/crates/router-core/src/cache.rs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
//! [`RouterCache`]: the DualCache-style store all routing state lives in, under the keys,
|
||||
//! encodings and TTLs the Python router uses, so Python and Rust workers share it. Method names
|
||||
//! follow `litellm-cache`'s `BaseCache`, `BatchCache`, `CounterCache` and `DeleteCache`.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use litellm_cache::Error;
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::Value;
|
||||
|
||||
#[async_trait]
|
||||
pub trait RouterCache: Send + Sync {
|
||||
async fn async_get_cache(&self, key: &str) -> Result<Option<Value>, Error>;
|
||||
|
||||
async fn async_set_cache(
|
||||
&self,
|
||||
key: &str,
|
||||
value: Value,
|
||||
ttl: Option<Duration>,
|
||||
) -> Result<(), Error>;
|
||||
|
||||
/// Add `amount` to the number at `key` (0 when missing) and return the new value. The TTL
|
||||
/// applies when the key is created, as Redis `INCRBYFLOAT` + `EXPIRE NX` does.
|
||||
async fn async_increment(
|
||||
&self,
|
||||
key: &str,
|
||||
amount: f64,
|
||||
ttl: Option<Duration>,
|
||||
) -> Result<f64, Error>;
|
||||
|
||||
async fn async_batch_get_cache(&self, keys: &[String]) -> Result<Vec<Option<Value>>, Error>;
|
||||
|
||||
async fn async_delete_cache(&self, key: &str) -> Result<(), Error>;
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Entry {
|
||||
value: Value,
|
||||
expires: Option<Instant>,
|
||||
}
|
||||
|
||||
/// An in-process [`RouterCache`], for tests and for state Python keeps `local_only`.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct InMemoryRouterCache {
|
||||
entries: Arc<Mutex<HashMap<String, Entry>>>,
|
||||
}
|
||||
|
||||
impl InMemoryRouterCache {
|
||||
fn live(entries: &mut HashMap<String, Entry>, key: &str) -> Option<Value> {
|
||||
let expired = entries
|
||||
.get(key)
|
||||
.is_some_and(|entry| entry.expires.is_some_and(|at| at <= Instant::now()));
|
||||
if expired {
|
||||
entries.remove(key);
|
||||
}
|
||||
entries.get(key).map(|entry| entry.value.clone())
|
||||
}
|
||||
|
||||
pub fn get(&self, key: &str) -> Option<Value> {
|
||||
Self::live(&mut self.entries.lock(), key)
|
||||
}
|
||||
|
||||
pub fn increment(&self, key: &str, amount: f64, ttl: Option<Duration>) -> f64 {
|
||||
let mut entries = self.entries.lock();
|
||||
let current = Self::live(&mut entries, key).and_then(|value| value.as_f64());
|
||||
let next = current.unwrap_or(0.0) + amount;
|
||||
let value = serde_json::Number::from_f64(next)
|
||||
.map(|number| {
|
||||
if next.fract() == 0.0 && next.abs() < i64::MAX as f64 {
|
||||
Value::from(next as i64)
|
||||
} else {
|
||||
Value::Number(number)
|
||||
}
|
||||
})
|
||||
.unwrap_or(Value::Null);
|
||||
let expires = match (current, entries.get(key)) {
|
||||
(Some(_), Some(entry)) => entry.expires,
|
||||
_ => ttl.map(|ttl| Instant::now() + ttl),
|
||||
};
|
||||
entries.insert(key.to_owned(), Entry { value, expires });
|
||||
next
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl RouterCache for InMemoryRouterCache {
|
||||
async fn async_get_cache(&self, key: &str) -> Result<Option<Value>, Error> {
|
||||
Ok(self.get(key))
|
||||
}
|
||||
|
||||
async fn async_set_cache(
|
||||
&self,
|
||||
key: &str,
|
||||
value: Value,
|
||||
ttl: Option<Duration>,
|
||||
) -> Result<(), Error> {
|
||||
let expires = ttl.map(|ttl| Instant::now() + ttl);
|
||||
self.entries
|
||||
.lock()
|
||||
.insert(key.to_owned(), Entry { value, expires });
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn async_increment(
|
||||
&self,
|
||||
key: &str,
|
||||
amount: f64,
|
||||
ttl: Option<Duration>,
|
||||
) -> Result<f64, Error> {
|
||||
Ok(self.increment(key, amount, ttl))
|
||||
}
|
||||
|
||||
async fn async_batch_get_cache(&self, keys: &[String]) -> Result<Vec<Option<Value>>, Error> {
|
||||
let mut entries = self.entries.lock();
|
||||
Ok(keys
|
||||
.iter()
|
||||
.map(|key| Self::live(&mut entries, key))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn async_delete_cache(&self, key: &str) -> Result<(), Error> {
|
||||
self.entries.lock().remove(key);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
592
litellm-rust/crates/router-core/src/call.rs
Normal file
592
litellm-rust/crates/router-core/src/call.rs
Normal file
|
|
@ -0,0 +1,592 @@
|
|||
//! The call loop: select, run the attempt, then retry in the same group or fall back to another
|
||||
//! until it succeeds or the policy is exhausted. Mirrors `Router.async_function_with_fallbacks`,
|
||||
//! `async_function_with_fallbacks_common_utils`, `async_function_with_retries`,
|
||||
//! `should_retry_this_error` and `_time_to_sleep_before_retry` in `litellm/router.py`, and
|
||||
//! `run_async_fallback` in `router_utils/fallback_event_handlers.py`.
|
||||
|
||||
use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc, time::Duration};
|
||||
|
||||
use tracing::warn;
|
||||
|
||||
use crate::{
|
||||
Core, Snapshot,
|
||||
config::Op,
|
||||
error::{AttemptError, ConfigError, Error, FailureKind},
|
||||
executor::Executor,
|
||||
health::{GroupShape, should_retry_status},
|
||||
host::{Attempt, Output, RouterEvent, RouterRequest},
|
||||
registry::ResolveKind,
|
||||
select::{Candidates, SelectInputs},
|
||||
};
|
||||
|
||||
/// `INITIAL_RETRY_DELAY`, `MAX_RETRY_DELAY` and `JITTER` in `litellm/constants.py`.
|
||||
const INITIAL_RETRY_DELAY: f64 = 0.5;
|
||||
const MAX_RETRY_DELAY: f64 = 8.0;
|
||||
const JITTER: f64 = 0.75;
|
||||
/// `calculate_retry_after` obeys a provider's retry-after up to a minute.
|
||||
const MAX_HONOURED_RETRY_AFTER: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct RouterResponse {
|
||||
pub output: Output,
|
||||
pub deployment_id: String,
|
||||
pub model_group: String,
|
||||
pub attempted_retries: u32,
|
||||
/// The retry budget of the loop a retry succeeded in (`x-litellm-max-retries`); `None`
|
||||
/// when the first attempt succeeded.
|
||||
pub max_retries: Option<u32>,
|
||||
pub attempted_fallbacks: u32,
|
||||
}
|
||||
|
||||
/// A failed attempt with the deployment it ran on, when one was picked.
|
||||
struct Failed {
|
||||
error: Error,
|
||||
deployment_id: Option<String>,
|
||||
}
|
||||
|
||||
impl From<Error> for Failed {
|
||||
fn from(error: Error) -> Self {
|
||||
Self {
|
||||
error,
|
||||
deployment_id: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a level of the fallback walk sits: `depth` is `run_async_fallback`'s counter, one more
|
||||
/// for every target tried, and `level` how many fallbacks deep the walk is.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
struct Hop {
|
||||
depth: u32,
|
||||
level: u32,
|
||||
}
|
||||
|
||||
/// A fallback target: a group, or the same group at a later `order`.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
|
||||
struct Target {
|
||||
group: String,
|
||||
order: Option<i64>,
|
||||
}
|
||||
|
||||
type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
/// `should_retry_this_error`, as a predicate.
|
||||
fn should_retry(
|
||||
failure: &AttemptError,
|
||||
healthy: usize,
|
||||
all: usize,
|
||||
has_context_window_fallbacks: bool,
|
||||
has_content_policy_fallbacks: bool,
|
||||
has_regular_fallbacks: bool,
|
||||
) -> bool {
|
||||
match failure.kind {
|
||||
FailureKind::ContextWindowExceeded if has_context_window_fallbacks => return false,
|
||||
FailureKind::ContentPolicyViolation if has_content_policy_fallbacks => return false,
|
||||
_ => {}
|
||||
}
|
||||
if let Some(status) = failure.status
|
||||
&& !should_retry_status(status)
|
||||
&& !matches!(status, 401 | 403)
|
||||
&& failure.kind != FailureKind::Declined
|
||||
{
|
||||
return false;
|
||||
}
|
||||
match failure.kind {
|
||||
FailureKind::NotFound => false,
|
||||
FailureKind::RateLimit if healthy == 0 && has_regular_fallbacks => false,
|
||||
FailureKind::Authentication | FailureKind::PermissionDenied if all <= 1 => false,
|
||||
_ => healthy > 0,
|
||||
}
|
||||
}
|
||||
|
||||
impl Core {
|
||||
pub(crate) async fn call(
|
||||
&self,
|
||||
op: Op,
|
||||
request: Arc<RouterRequest>,
|
||||
) -> Result<RouterResponse, Error> {
|
||||
let snapshot = self.snapshot.load_full();
|
||||
let mut attempted = HashSet::new();
|
||||
let target = Target {
|
||||
group: request.model.clone(),
|
||||
order: None,
|
||||
};
|
||||
self.with_fallbacks(
|
||||
&snapshot,
|
||||
&op,
|
||||
&request,
|
||||
target,
|
||||
Hop::default(),
|
||||
&mut attempted,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// One level of `async_function_with_fallbacks`: the target's retry loop, then its
|
||||
/// fallbacks. A level that fails raises its own error, whatever its fallbacks raised.
|
||||
fn with_fallbacks<'a>(
|
||||
&'a self,
|
||||
snapshot: &'a Snapshot,
|
||||
op: &'a Op,
|
||||
request: &'a Arc<RouterRequest>,
|
||||
target: Target,
|
||||
at: Hop,
|
||||
attempted: &'a mut HashSet<Target>,
|
||||
) -> BoxFuture<'a, Result<RouterResponse, Error>> {
|
||||
Box::pin(async move {
|
||||
let error = match self.with_retries(snapshot, op, request, &target, at).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) => error,
|
||||
};
|
||||
let Some(targets) = self.fallback_targets(snapshot, request, &target, &error) else {
|
||||
return Err(error);
|
||||
};
|
||||
// `run_async_fallback`: the depth is checked once on entry, then every target
|
||||
// tried at this level counts one more.
|
||||
if at.depth >= snapshot.settings.max_fallbacks {
|
||||
return Err(error);
|
||||
}
|
||||
attempted.insert(Target {
|
||||
group: target.group.clone(),
|
||||
order: None,
|
||||
});
|
||||
let mut hop = at.depth;
|
||||
for next in targets {
|
||||
if next.order.is_none() && next.group == target.group {
|
||||
continue;
|
||||
}
|
||||
if next.group != target.group
|
||||
&& !self.host.allow_fallback(request, &next.group).await
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if !attempted.insert(next.clone()) {
|
||||
continue;
|
||||
}
|
||||
hop += 1;
|
||||
self.host.observe(&RouterEvent::Fallback {
|
||||
from: target.group.clone(),
|
||||
to: next.group.clone(),
|
||||
order: next.order,
|
||||
depth: hop,
|
||||
level: at.level + 1,
|
||||
});
|
||||
if let Ok(mut response) = self
|
||||
.with_fallbacks(
|
||||
snapshot,
|
||||
op,
|
||||
request,
|
||||
next,
|
||||
Hop {
|
||||
depth: hop,
|
||||
level: at.level + 1,
|
||||
},
|
||||
attempted,
|
||||
)
|
||||
.await
|
||||
{
|
||||
response.attempted_fallbacks = hop;
|
||||
return Ok(response);
|
||||
}
|
||||
}
|
||||
Err(error)
|
||||
})
|
||||
}
|
||||
|
||||
/// The fallback targets for a failure, or `None` when it must be raised as is
|
||||
/// (`async_function_with_fallbacks_common_utils`).
|
||||
fn fallback_targets(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
request: &RouterRequest,
|
||||
target: &Target,
|
||||
error: &Error,
|
||||
) -> Option<Vec<Target>> {
|
||||
if request.disable_fallbacks {
|
||||
return None;
|
||||
}
|
||||
let kind = match error {
|
||||
Error::Attempt(failure) => failure.kind,
|
||||
Error::NoHealthyDeployments { .. } => FailureKind::RateLimit,
|
||||
Error::Config(_) | Error::UnknownModel { .. } | Error::Cache(_) | Error::Retired => {
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if kind == FailureKind::GuardrailIntervention {
|
||||
return None;
|
||||
}
|
||||
let dynamic = &snapshot.settings.dynamic;
|
||||
// `fallback_lookup_groups`: the group being served, then the group the request was for.
|
||||
let mut lookup = vec![target.group.as_str()];
|
||||
if request.model != target.group {
|
||||
lookup.push(request.model.as_str());
|
||||
}
|
||||
let groups = |names: Vec<String>| {
|
||||
names
|
||||
.into_iter()
|
||||
.map(|group| Target { group, order: None })
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let skip_order = matches!(
|
||||
kind,
|
||||
FailureKind::ContextWindowExceeded | FailureKind::ContentPolicyViolation
|
||||
);
|
||||
if !skip_order {
|
||||
let mut orders: Vec<i64> = snapshot
|
||||
.registry
|
||||
.group(&target.group)
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.filter(|d| d.usable_by_team(request.team_id.as_deref()))
|
||||
.filter_map(|d| d.order())
|
||||
.collect();
|
||||
orders.sort_unstable();
|
||||
orders.dedup();
|
||||
if orders.len() > 1 {
|
||||
let current = target.order.unwrap_or(orders[0]);
|
||||
let mut targets: Vec<Target> = orders
|
||||
.into_iter()
|
||||
.filter(|order| *order > current)
|
||||
.map(|order| Target {
|
||||
group: target.group.clone(),
|
||||
order: Some(order),
|
||||
})
|
||||
.collect();
|
||||
targets.extend(groups(
|
||||
dynamic
|
||||
.fallbacks
|
||||
.lookup_groups(&lookup, &dynamic.fallback_keys)
|
||||
.unwrap_or_default(),
|
||||
));
|
||||
if !targets.is_empty() {
|
||||
return Some(targets);
|
||||
}
|
||||
}
|
||||
}
|
||||
let specific = match kind {
|
||||
FailureKind::ContextWindowExceeded => dynamic.context_window_fallbacks.as_ref(),
|
||||
FailureKind::ContentPolicyViolation => dynamic.content_policy_fallbacks.as_ref(),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(fallbacks) = specific {
|
||||
return fallbacks
|
||||
.lookup_groups(&lookup, &dynamic.fallback_keys)
|
||||
.map(groups);
|
||||
}
|
||||
dynamic.fallback_targets(&lookup).map(groups)
|
||||
}
|
||||
|
||||
async fn with_retries(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
op: &Op,
|
||||
request: &Arc<RouterRequest>,
|
||||
target: &Target,
|
||||
at: Hop,
|
||||
) -> Result<RouterResponse, Error> {
|
||||
let mut inputs = SelectInputs {
|
||||
target_order: target.order,
|
||||
..SelectInputs::default()
|
||||
};
|
||||
let counting = SelectInputs {
|
||||
count_only: true,
|
||||
..SelectInputs::default()
|
||||
};
|
||||
let first = self
|
||||
.attempt(snapshot, op, request, target, &inputs, 0, at)
|
||||
.await;
|
||||
let mut failed = match first {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(failed) => failed,
|
||||
};
|
||||
let Error::Attempt(failure) = &failed.error else {
|
||||
return Err(failed.error);
|
||||
};
|
||||
if failure.kind == FailureKind::GuardrailIntervention {
|
||||
return Err(failed.error);
|
||||
}
|
||||
let settings = &snapshot.settings;
|
||||
let mut num_retries = request
|
||||
.num_retries
|
||||
.or(failure.num_retries)
|
||||
.unwrap_or(settings.num_retries);
|
||||
let policy_retries = (request.num_retries != Some(0))
|
||||
.then(|| settings.retry_policy.as_ref()?.retries_for(failure.kind))
|
||||
.flatten();
|
||||
let policy_applies = policy_retries.is_some();
|
||||
num_retries = policy_retries.unwrap_or(num_retries);
|
||||
// `_async_get_healthy_deployments` finds no group for a deployment id, so a call by id
|
||||
// counts no deployments when deciding whether to retry.
|
||||
let counted = |c: &Candidates, n: usize| if c.kind == ResolveKind::Id { 0 } else { n };
|
||||
let all = self
|
||||
.candidates(snapshot, op, request, &target.group, &counting)
|
||||
.await
|
||||
.map(|c| counted(&c, c.all.len()))
|
||||
.unwrap_or(0);
|
||||
|
||||
for retry in 0..=num_retries {
|
||||
let Error::Attempt(failure) = &failed.error else {
|
||||
return Err(failed.error);
|
||||
};
|
||||
let healthy = self
|
||||
.candidates(snapshot, op, request, &target.group, &counting)
|
||||
.await
|
||||
.map(|c| counted(&c, c.healthy.len()))
|
||||
.unwrap_or(0);
|
||||
if !policy_applies && !self.should_retry(snapshot, failure, healthy, all) {
|
||||
return Err(failed.error);
|
||||
}
|
||||
if num_retries == 0 {
|
||||
return Err(failed.error);
|
||||
}
|
||||
if let (Some(status), Some(id)) = (failure.status, &failed.deployment_id)
|
||||
&& !should_retry_status(status)
|
||||
&& !inputs.skipped_ids.contains(id)
|
||||
{
|
||||
inputs.skipped_ids.push(id.clone());
|
||||
}
|
||||
let delay = self.time_to_sleep(
|
||||
snapshot,
|
||||
failure,
|
||||
num_retries - retry,
|
||||
num_retries,
|
||||
healthy,
|
||||
all,
|
||||
);
|
||||
self.host.sleep(delay).await;
|
||||
if retry == num_retries {
|
||||
self.host.observe(&RouterEvent::RetriesExhausted {
|
||||
model_group: target.group.clone(),
|
||||
attempted: num_retries,
|
||||
max: num_retries,
|
||||
});
|
||||
break;
|
||||
}
|
||||
self.host.observe(&RouterEvent::Retry {
|
||||
model_group: target.group.clone(),
|
||||
retry: retry + 1,
|
||||
delay,
|
||||
});
|
||||
match self
|
||||
.attempt(snapshot, op, request, target, &inputs, retry + 1, at)
|
||||
.await
|
||||
{
|
||||
Ok(mut response) => {
|
||||
response.max_retries = Some(num_retries);
|
||||
return Ok(response);
|
||||
}
|
||||
Err(next) => failed = next,
|
||||
}
|
||||
}
|
||||
Err(failed.error)
|
||||
}
|
||||
|
||||
fn should_retry(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
failure: &AttemptError,
|
||||
healthy: usize,
|
||||
all: usize,
|
||||
) -> bool {
|
||||
let dynamic = &snapshot.settings.dynamic;
|
||||
should_retry(
|
||||
failure,
|
||||
healthy,
|
||||
all,
|
||||
dynamic.context_window_fallbacks.is_some(),
|
||||
dynamic.content_policy_fallbacks.is_some(),
|
||||
dynamic.has_regular_fallbacks(),
|
||||
)
|
||||
}
|
||||
|
||||
/// `_time_to_sleep_before_retry` and `calculate_retry_after`.
|
||||
fn time_to_sleep(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
failure: &AttemptError,
|
||||
remaining_retries: u32,
|
||||
num_retries: u32,
|
||||
healthy: usize,
|
||||
all: usize,
|
||||
) -> Duration {
|
||||
if all != 1 && healthy > 0 {
|
||||
return Duration::ZERO;
|
||||
}
|
||||
let jitter = JITTER * self.rng.next_f64();
|
||||
if let Some(retry_after) = failure.retry_after
|
||||
&& !retry_after.is_zero()
|
||||
&& retry_after <= MAX_HONOURED_RETRY_AFTER
|
||||
{
|
||||
return retry_after + Duration::from_secs_f64(jitter);
|
||||
}
|
||||
let exponent = num_retries.saturating_sub(remaining_retries);
|
||||
let backoff = (INITIAL_RETRY_DELAY * 2f64.powi(exponent as i32))
|
||||
.max(snapshot.settings.retry_after.as_secs_f64())
|
||||
.min(MAX_RETRY_DELAY);
|
||||
Duration::from_secs_f64(backoff + jitter)
|
||||
}
|
||||
|
||||
/// One attempt: select a deployment, run it on its executor, record the outcome.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn attempt(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
op: &Op,
|
||||
request: &Arc<RouterRequest>,
|
||||
target: &Target,
|
||||
inputs: &SelectInputs,
|
||||
retry: u32,
|
||||
at: Hop,
|
||||
) -> Result<RouterResponse, Failed> {
|
||||
let candidates = self
|
||||
.candidates(snapshot, op, request, &target.group, inputs)
|
||||
.await?;
|
||||
if candidates.healthy.is_empty() {
|
||||
return Err(Error::NoHealthyDeployments {
|
||||
model: candidates.model_group,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let deployment = self.pick(snapshot, op, request, &candidates).await?;
|
||||
self.host.observe(&RouterEvent::Selected {
|
||||
model_group: candidates.model_group.clone(),
|
||||
deployment_id: deployment.id.clone(),
|
||||
});
|
||||
let executor = self
|
||||
.executors
|
||||
.resolve(op, deployment.provider())
|
||||
.map_err(|unsupported| Error::Config(ConfigError::Unsupported(unsupported)))?;
|
||||
let attempt = Attempt {
|
||||
op: op.clone(),
|
||||
request: request.clone(),
|
||||
deployment: deployment.clone(),
|
||||
name: target.group.clone(),
|
||||
model_group: candidates.model_group.clone(),
|
||||
target_order: target.order,
|
||||
candidate_ids: candidates.all.iter().map(|d| d.id.clone()).collect(),
|
||||
retry,
|
||||
fallback_depth: at.depth,
|
||||
fallback_level: at.level,
|
||||
timeout: snapshot.settings.timeout,
|
||||
};
|
||||
let (result, recorded) = match executor {
|
||||
Executor::Native(route) => (route.execute(&attempt).await, false),
|
||||
Executor::Host => (
|
||||
self.host.invoke(&attempt).await,
|
||||
self.host.records_outcomes(),
|
||||
),
|
||||
};
|
||||
match result {
|
||||
Ok(output) => {
|
||||
if !recorded
|
||||
&& let Err(error) = self
|
||||
.health
|
||||
.record_success(&deployment, output.total_tokens)
|
||||
.await
|
||||
{
|
||||
warn!(deployment_id = %deployment.id, error = %error, "usage was not recorded");
|
||||
}
|
||||
self.host.observe(&RouterEvent::Succeeded {
|
||||
deployment_id: deployment.id.clone(),
|
||||
});
|
||||
Ok(RouterResponse {
|
||||
output,
|
||||
deployment_id: deployment.id.clone(),
|
||||
model_group: candidates.model_group,
|
||||
attempted_retries: retry,
|
||||
max_retries: None,
|
||||
attempted_fallbacks: at.depth,
|
||||
})
|
||||
}
|
||||
Err(failure) => {
|
||||
if recorded {
|
||||
self.host.observe(&RouterEvent::AttemptFailed {
|
||||
deployment_id: deployment.id.clone(),
|
||||
kind: failure.kind,
|
||||
status: failure.status,
|
||||
});
|
||||
} else {
|
||||
self.record_failure(snapshot, &candidates, &deployment, &failure)
|
||||
.await;
|
||||
}
|
||||
Err(Failed {
|
||||
error: Error::Attempt(failure),
|
||||
deployment_id: Some(deployment.id.clone()),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_failure(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
candidates: &Candidates,
|
||||
deployment: &crate::registry::Deployment,
|
||||
failure: &AttemptError,
|
||||
) {
|
||||
self.host.observe(&RouterEvent::AttemptFailed {
|
||||
deployment_id: deployment.id.clone(),
|
||||
kind: failure.kind,
|
||||
status: failure.status,
|
||||
});
|
||||
let shape = GroupShape {
|
||||
single_deployment: candidates.all.len() == 1,
|
||||
has_fallback_group: snapshot
|
||||
.settings
|
||||
.dynamic
|
||||
.fallback_targets(&[candidates.model_group.as_str()])
|
||||
.is_some(),
|
||||
};
|
||||
match self
|
||||
.health
|
||||
.record_failure(&snapshot.settings, deployment, shape, failure)
|
||||
.await
|
||||
{
|
||||
Ok(Some(cooldown)) => self.host.observe(&RouterEvent::CooldownSet {
|
||||
deployment_id: deployment.id.clone(),
|
||||
cooldown,
|
||||
}),
|
||||
Ok(None) => {}
|
||||
Err(error) => {
|
||||
warn!(deployment_id = %deployment.id, error = %error, "cooldown was not recorded");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::rate_limit_with_healthy(FailureKind::RateLimit, Some(429), 1, 2, false, true)]
|
||||
#[case::rate_limit_none_healthy_with_fallbacks(
|
||||
FailureKind::RateLimit,
|
||||
Some(429),
|
||||
0,
|
||||
2,
|
||||
true,
|
||||
false
|
||||
)]
|
||||
#[case::bad_request(FailureKind::BadRequest, Some(400), 2, 2, false, false)]
|
||||
#[case::auth_lone_deployment(FailureKind::Authentication, Some(401), 1, 1, false, false)]
|
||||
#[case::auth_with_siblings(FailureKind::Authentication, Some(401), 1, 2, false, true)]
|
||||
#[case::not_found(FailureKind::NotFound, Some(404), 2, 2, false, false)]
|
||||
#[case::server_error_no_healthy(FailureKind::InternalServer, Some(500), 0, 2, false, false)]
|
||||
fn should_retry_follows_python(
|
||||
#[case] kind: FailureKind,
|
||||
#[case] status: Option<u16>,
|
||||
#[case] healthy: usize,
|
||||
#[case] all: usize,
|
||||
#[case] regular_fallbacks: bool,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
let failure = AttemptError::new(kind, status);
|
||||
assert_eq!(
|
||||
should_retry(&failure, healthy, all, false, false, regular_fallbacks),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
433
litellm-rust/crates/router-core/src/config.rs
Normal file
433
litellm-rust/crates/router-core/src/config.rs
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
//! What the router is told: [`RouterConfig`], fixed for the router's life, and
|
||||
//! [`RouterDynamicConfig`], a versioned snapshot read once per call. The router never reads
|
||||
//! process globals.
|
||||
|
||||
use std::{collections::BTreeMap, time::Duration};
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
error::FailureKind,
|
||||
registry::{DeploymentSpec, RoutingGroupSpec},
|
||||
};
|
||||
|
||||
/// `DEFAULT_COOLDOWN_TIME_SECONDS` in `litellm/constants.py`.
|
||||
pub const DEFAULT_COOLDOWN_TIME: Duration = Duration::from_secs(5);
|
||||
/// `ROUTER_MAX_FALLBACKS` in `litellm/constants.py`.
|
||||
pub const DEFAULT_MAX_FALLBACKS: u32 = 5;
|
||||
|
||||
/// The router operation a call runs, spelled as the Python router's endpoint method.
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Hash,
|
||||
strum::EnumString,
|
||||
strum::Display,
|
||||
serde_with::DeserializeFromStr,
|
||||
)]
|
||||
pub enum Op {
|
||||
#[strum(serialize = "acompletion")]
|
||||
Completion,
|
||||
#[strum(serialize = "atext_completion")]
|
||||
TextCompletion,
|
||||
#[strum(serialize = "aembedding")]
|
||||
Embedding,
|
||||
#[strum(serialize = "aresponses")]
|
||||
Responses,
|
||||
#[strum(serialize = "aanthropic_messages")]
|
||||
Messages,
|
||||
#[strum(serialize = "aimage_generation")]
|
||||
ImageGeneration,
|
||||
#[strum(serialize = "atranscription")]
|
||||
Transcription,
|
||||
#[strum(serialize = "aspeech")]
|
||||
Speech,
|
||||
#[strum(serialize = "aocr")]
|
||||
Ocr,
|
||||
#[strum(default, transparent)]
|
||||
Other(String),
|
||||
}
|
||||
|
||||
/// `routing_strategy`. Only simple-shuffle is native; the rest go through `Host::select`.
|
||||
#[derive(
|
||||
Clone,
|
||||
Debug,
|
||||
Default,
|
||||
PartialEq,
|
||||
Eq,
|
||||
strum::EnumString,
|
||||
strum::Display,
|
||||
serde_with::DeserializeFromStr,
|
||||
)]
|
||||
pub enum RoutingStrategy {
|
||||
#[default]
|
||||
#[strum(serialize = "simple-shuffle")]
|
||||
SimpleShuffle,
|
||||
#[strum(serialize = "least-busy")]
|
||||
LeastBusy,
|
||||
#[strum(serialize = "usage-based-routing")]
|
||||
UsageBased,
|
||||
#[strum(serialize = "usage-based-routing-v2")]
|
||||
UsageBasedV2,
|
||||
#[strum(serialize = "latency-based-routing")]
|
||||
LatencyBased,
|
||||
#[strum(serialize = "cost-based-routing")]
|
||||
CostBased,
|
||||
#[strum(default, transparent)]
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl RoutingStrategy {
|
||||
pub fn is_native(&self) -> bool {
|
||||
matches!(self, Self::SimpleShuffle)
|
||||
}
|
||||
}
|
||||
|
||||
/// `PluginKind` in `litellm/router_api.py`.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::Display)]
|
||||
pub enum PluginKind {
|
||||
#[serde(rename = "auto")]
|
||||
#[strum(serialize = "auto")]
|
||||
Auto,
|
||||
#[serde(rename = "complexity")]
|
||||
#[strum(serialize = "complexity")]
|
||||
Complexity,
|
||||
#[serde(rename = "quality")]
|
||||
#[strum(serialize = "quality")]
|
||||
Quality,
|
||||
#[serde(rename = "adaptive")]
|
||||
#[strum(serialize = "adaptive")]
|
||||
Adaptive,
|
||||
#[serde(rename = "routing")]
|
||||
#[strum(serialize = "routing")]
|
||||
Routing,
|
||||
#[serde(rename = "budget_limiter")]
|
||||
#[strum(serialize = "budget_limiter")]
|
||||
BudgetLimiter,
|
||||
#[serde(rename = "prompt_caching_pins")]
|
||||
#[strum(serialize = "prompt_caching_pins")]
|
||||
PromptCachingPins,
|
||||
}
|
||||
|
||||
/// One registered plugin. Its behaviour lives in the host until it is ported.
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize)]
|
||||
pub struct PluginSpec {
|
||||
pub kind: PluginKind,
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub config: serde_json::Value,
|
||||
}
|
||||
|
||||
/// `RetryPolicy` in `litellm/types/router.py`: retries per exception type.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
#[allow(non_snake_case)]
|
||||
pub struct RetryPolicy {
|
||||
pub BadRequestErrorRetries: Option<u32>,
|
||||
pub AuthenticationErrorRetries: Option<u32>,
|
||||
pub TimeoutErrorRetries: Option<u32>,
|
||||
pub RateLimitErrorRetries: Option<u32>,
|
||||
pub ContentPolicyViolationErrorRetries: Option<u32>,
|
||||
pub InternalServerErrorRetries: Option<u32>,
|
||||
}
|
||||
|
||||
impl RetryPolicy {
|
||||
pub fn retries_for(&self, kind: FailureKind) -> Option<u32> {
|
||||
match kind {
|
||||
FailureKind::BadRequest | FailureKind::ContextWindowExceeded => {
|
||||
self.BadRequestErrorRetries
|
||||
}
|
||||
FailureKind::Authentication => self.AuthenticationErrorRetries,
|
||||
FailureKind::Timeout => self.TimeoutErrorRetries,
|
||||
FailureKind::RateLimit => self.RateLimitErrorRetries,
|
||||
FailureKind::ContentPolicyViolation => self.ContentPolicyViolationErrorRetries,
|
||||
FailureKind::InternalServer => self.InternalServerErrorRetries,
|
||||
FailureKind::PermissionDenied
|
||||
| FailureKind::NotFound
|
||||
| FailureKind::ApiConnection
|
||||
| FailureKind::ServiceUnavailable
|
||||
| FailureKind::BadGateway
|
||||
| FailureKind::GuardrailIntervention
|
||||
| FailureKind::Declined
|
||||
| FailureKind::CallerCredential
|
||||
| FailureKind::Other => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `AllowedFailsPolicy` in `litellm/types/router.py`: allowed fails per minute per exception type.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
#[allow(non_snake_case)]
|
||||
pub struct AllowedFailsPolicy {
|
||||
pub BadRequestErrorAllowedFails: Option<u32>,
|
||||
pub AuthenticationErrorAllowedFails: Option<u32>,
|
||||
pub TimeoutErrorAllowedFails: Option<u32>,
|
||||
pub RateLimitErrorAllowedFails: Option<u32>,
|
||||
pub ContentPolicyViolationErrorAllowedFails: Option<u32>,
|
||||
pub InternalServerErrorAllowedFails: Option<u32>,
|
||||
pub ServiceUnavailableErrorAllowedFails: Option<u32>,
|
||||
pub BadGatewayErrorAllowedFails: Option<u32>,
|
||||
}
|
||||
|
||||
impl AllowedFailsPolicy {
|
||||
pub fn allowed_fails_for(&self, kind: FailureKind) -> Option<u32> {
|
||||
match kind {
|
||||
FailureKind::BadRequest | FailureKind::ContextWindowExceeded => {
|
||||
self.BadRequestErrorAllowedFails
|
||||
}
|
||||
FailureKind::Authentication => self.AuthenticationErrorAllowedFails,
|
||||
FailureKind::Timeout => self.TimeoutErrorAllowedFails,
|
||||
FailureKind::RateLimit => self.RateLimitErrorAllowedFails,
|
||||
FailureKind::ContentPolicyViolation => self.ContentPolicyViolationErrorAllowedFails,
|
||||
FailureKind::InternalServer => self.InternalServerErrorAllowedFails,
|
||||
FailureKind::ServiceUnavailable => self.ServiceUnavailableErrorAllowedFails,
|
||||
FailureKind::BadGateway => self.BadGatewayErrorAllowedFails,
|
||||
FailureKind::PermissionDenied
|
||||
| FailureKind::NotFound
|
||||
| FailureKind::ApiConnection
|
||||
| FailureKind::GuardrailIntervention
|
||||
| FailureKind::Declined
|
||||
| FailureKind::CallerCredential
|
||||
| FailureKind::Other => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A fallback list as Python spells it, `[{"gpt-4": ["gpt-3.5"]}, {"*": ["default"]}]`, kept in
|
||||
/// order. `"*"` is the default entry.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
#[serde(from = "Vec<BTreeMap<String, Vec<String>>>")]
|
||||
pub struct Fallbacks(Vec<(String, Vec<String>)>);
|
||||
|
||||
impl From<Vec<BTreeMap<String, Vec<String>>>> for Fallbacks {
|
||||
fn from(entries: Vec<BTreeMap<String, Vec<String>>>) -> Self {
|
||||
Self(entries.into_iter().flatten().collect())
|
||||
}
|
||||
}
|
||||
|
||||
impl Fallbacks {
|
||||
pub fn new(entries: impl IntoIterator<Item = (String, Vec<String>)>) -> Self {
|
||||
Self(entries.into_iter().collect())
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.0.is_empty()
|
||||
}
|
||||
|
||||
/// `_get_first_default_fallback`: the first target of the first `"*"` entry.
|
||||
pub fn default_target(&self) -> Option<&str> {
|
||||
self.0
|
||||
.iter()
|
||||
.find(|(key, targets)| key == "*" && !targets.is_empty())
|
||||
.map(|(_, targets)| targets[0].as_str())
|
||||
}
|
||||
|
||||
/// `get_fallback_model_group`: the group's own entry, else an entry keyed by a
|
||||
/// provider-prefixed or provider-stripped spelling of it (the last such entry), else the
|
||||
/// last `"*"` entry before the group's own. The flag says the targets came from `"*"`.
|
||||
pub fn lookup(&self, group: &str, keys: &FallbackKeys) -> Option<(&[String], bool)> {
|
||||
let prefixed = keys.provider_prefixed.get(group).map(String::as_str);
|
||||
let mut spelled = None;
|
||||
let mut generic = None;
|
||||
for (key, targets) in &self.0 {
|
||||
if key == group {
|
||||
return Some((targets, false));
|
||||
}
|
||||
if Some(key.as_str()) == prefixed || keys.is_stripped(group, key) {
|
||||
spelled = Some(targets.as_slice());
|
||||
} else if key == "*" {
|
||||
generic = Some(targets.as_slice());
|
||||
}
|
||||
}
|
||||
spelled
|
||||
.map(|targets| (targets, false))
|
||||
.or_else(|| generic.map(|targets| (targets, true)))
|
||||
}
|
||||
|
||||
/// `get_fallback_model_group_for_lookup_groups`: the first lookup group with its own
|
||||
/// chain wins; `"*"` applies only after every group missed.
|
||||
pub fn lookup_groups(&self, groups: &[&str], keys: &FallbackKeys) -> Option<Vec<String>> {
|
||||
let results: Vec<_> = groups
|
||||
.iter()
|
||||
.map(|group| self.lookup(group, keys))
|
||||
.collect();
|
||||
results
|
||||
.iter()
|
||||
.flatten()
|
||||
.find(|(_, generic)| !generic)
|
||||
.or_else(|| results.iter().flatten().next())
|
||||
.map(|(targets, _)| targets.to_vec())
|
||||
}
|
||||
}
|
||||
|
||||
/// What fallback keys match besides the exact group name: Python's `provider_list`, for
|
||||
/// `"openai/gpt-4o"` to use a `"gpt-4o"` entry, and the provider-prefixed spelling of each
|
||||
/// bare group name (`_provider_prefixed_model_group`), for `"gpt-4o"` to use an
|
||||
/// `"openai/gpt-4o"` entry.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct FallbackKeys {
|
||||
pub providers: Vec<String>,
|
||||
pub provider_prefixed: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl FallbackKeys {
|
||||
/// `_check_stripped_model_group`.
|
||||
fn is_stripped(&self, group: &str, key: &str) -> bool {
|
||||
self.providers.iter().any(|provider| {
|
||||
let prefix = format!("{provider}/");
|
||||
group.starts_with(&prefix) && group.replace(&prefix, "") == key
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Static config, fixed for the router's life.
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct RouterConfig {
|
||||
pub model_list: Vec<DeploymentSpec>,
|
||||
pub routing_strategy: RoutingStrategy,
|
||||
pub routing_groups: Vec<RoutingGroupSpec>,
|
||||
pub plugins: Vec<PluginSpec>,
|
||||
/// Router default `cooldown_time`; `None` is `DEFAULT_COOLDOWN_TIME_SECONDS`.
|
||||
pub cooldown_time: Option<f64>,
|
||||
pub disable_cooldowns: bool,
|
||||
pub allowed_fails_policy: Option<AllowedFailsPolicy>,
|
||||
pub retry_policy: Option<RetryPolicy>,
|
||||
/// `retry_after`: the minimum backoff between retries, in seconds.
|
||||
pub retry_after: f64,
|
||||
pub timeout: Option<f64>,
|
||||
pub request_timeout: Option<f64>,
|
||||
pub drop_params: Option<bool>,
|
||||
pub enable_pre_call_checks: bool,
|
||||
pub enable_health_check_routing: bool,
|
||||
/// Deployment ids that are a provider's default and never cool down
|
||||
/// (`provider_default_deployment_ids`).
|
||||
pub provider_default_deployment_ids: Vec<String>,
|
||||
/// The operations this router must serve; with a host that cannot invoke, each must have a
|
||||
/// native executor for every configured provider or the router fails to build.
|
||||
pub served_ops: Vec<Op>,
|
||||
}
|
||||
|
||||
/// Dynamic config: a versioned snapshot read once per call.
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct RouterDynamicConfig {
|
||||
pub version: u64,
|
||||
/// `model_group_alias` as Python spells it: each alias maps to a group name, or to
|
||||
/// `{"model": <group>, "hidden": <bool>}`.
|
||||
#[serde(alias = "model_group_alias", deserialize_with = "alias_targets")]
|
||||
pub model_alias_map: BTreeMap<String, String>,
|
||||
pub fallbacks: Fallbacks,
|
||||
pub default_fallbacks: Vec<String>,
|
||||
pub context_window_fallbacks: Option<Fallbacks>,
|
||||
pub content_policy_fallbacks: Option<Fallbacks>,
|
||||
pub num_retries: Option<u32>,
|
||||
pub max_fallbacks: u32,
|
||||
/// `allowed_fails`. `Some` means it was set explicitly, which switches cooldowns from the
|
||||
/// failure-ratio rule to the allowed-fails counter (`_is_allowed_fails_set_on_router`).
|
||||
pub allowed_fails: Option<u32>,
|
||||
/// Other spellings a fallback key matches.
|
||||
pub fallback_keys: FallbackKeys,
|
||||
}
|
||||
|
||||
/// One `model_group_alias` value: a group name, or `{"model": <group>, "hidden": <bool>}`.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum AliasTarget {
|
||||
Group(String),
|
||||
Entry { model: String },
|
||||
}
|
||||
|
||||
fn alias_targets<'de, D>(deserializer: D) -> Result<BTreeMap<String, String>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let aliases = BTreeMap::<String, AliasTarget>::deserialize(deserializer)?;
|
||||
Ok(aliases
|
||||
.into_iter()
|
||||
.map(|(alias, target)| match target {
|
||||
AliasTarget::Group(group) | AliasTarget::Entry { model: group } => (alias, group),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
impl Default for RouterDynamicConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: 0,
|
||||
model_alias_map: BTreeMap::new(),
|
||||
fallbacks: Fallbacks::default(),
|
||||
default_fallbacks: Vec::new(),
|
||||
context_window_fallbacks: None,
|
||||
content_policy_fallbacks: None,
|
||||
num_retries: None,
|
||||
max_fallbacks: DEFAULT_MAX_FALLBACKS,
|
||||
allowed_fails: None,
|
||||
fallback_keys: FallbackKeys::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RouterDynamicConfig {
|
||||
/// The regular fallback targets for the lookup groups (`get_fallback_model_group_for_lookup_groups`
|
||||
/// over `fallbacks`), then `default_fallbacks`, which Python folds into `fallbacks` as
|
||||
/// `{"*": [...]}`.
|
||||
pub fn fallback_targets(&self, groups: &[&str]) -> Option<Vec<String>> {
|
||||
self.fallbacks
|
||||
.lookup_groups(groups, &self.fallback_keys)
|
||||
.or_else(|| {
|
||||
(!self.default_fallbacks.is_empty()).then(|| self.default_fallbacks.clone())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn has_regular_fallbacks(&self) -> bool {
|
||||
!self.fallbacks.is_empty() || !self.default_fallbacks.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::own_entry("gpt-4", Some(vec!["gpt-3.5".to_owned()]))]
|
||||
#[case::star_entry("other", Some(vec!["default".to_owned()]))]
|
||||
fn fallback_targets_prefer_the_groups_own_entry(
|
||||
#[case] group: &str,
|
||||
#[case] expected: Option<Vec<String>>,
|
||||
) {
|
||||
let dynamic: RouterDynamicConfig = serde_json::from_value(serde_json::json!({
|
||||
"fallbacks": [{"gpt-4": ["gpt-3.5"]}, {"*": ["default"]}]
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(dynamic.fallback_targets(&[group]), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::group_name(json!({"model_group_alias": {"gpt": "gpt-4"}}))]
|
||||
#[case::hidden_entry(json!({"model_group_alias": {"gpt": {"model": "gpt-4", "hidden": true}}}))]
|
||||
#[case::native_name(json!({"model_alias_map": {"gpt": "gpt-4"}}))]
|
||||
fn model_group_alias_reads_both_python_spellings(#[case] wire: serde_json::Value) {
|
||||
let dynamic: RouterDynamicConfig = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(
|
||||
dynamic.model_alias_map,
|
||||
BTreeMap::from([("gpt".to_owned(), "gpt-4".to_owned())])
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::native("simple-shuffle", RoutingStrategy::SimpleShuffle)]
|
||||
#[case::unknown("my-strategy", RoutingStrategy::Other("my-strategy".to_owned()))]
|
||||
fn routing_strategy_parses_its_wire_name(
|
||||
#[case] wire: &str,
|
||||
#[case] expected: RoutingStrategy,
|
||||
) {
|
||||
assert_eq!(wire.parse::<RoutingStrategy>().unwrap(), expected);
|
||||
assert_eq!(expected.to_string(), wire);
|
||||
}
|
||||
}
|
||||
190
litellm-rust/crates/router-core/src/error.rs
Normal file
190
litellm-rust/crates/router-core/src/error.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
//! The router's errors: [`ConfigError`] when a router is built or changed, [`Error`] when a call
|
||||
//! or a selection fails, and [`AttemptError`], the typed facts of one failed attempt.
|
||||
|
||||
use std::{any::Any, fmt, sync::Arc, time::Duration};
|
||||
|
||||
use crate::config::{Op, RoutingStrategy};
|
||||
|
||||
/// What a host cannot serve, found when the router is built rather than partway through a call.
|
||||
#[derive(Clone, Debug, PartialEq, thiserror::Error)]
|
||||
pub enum Unsupported {
|
||||
#[error("routing strategy `{0}` needs a host that can select")]
|
||||
Strategy(RoutingStrategy),
|
||||
#[error("routing plugin `{0}` needs a host that can select")]
|
||||
Plugin(String),
|
||||
#[error("operation `{op}` on provider `{provider}` has no executor")]
|
||||
Executor { op: Op, provider: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ConfigError {
|
||||
#[error("the router config asks for something this host cannot serve")]
|
||||
Unsupported(#[from] Unsupported),
|
||||
#[error("deployment `{model_name}` has invalid litellm_params")]
|
||||
InvalidDeployment {
|
||||
model_name: String,
|
||||
#[source]
|
||||
source: serde_json::Error,
|
||||
},
|
||||
#[error("deployment id `{id}` is configured more than once")]
|
||||
DuplicateId { id: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("the router config is invalid")]
|
||||
Config(#[from] ConfigError),
|
||||
#[error("no deployment serves model `{model}`")]
|
||||
UnknownModel { model: String },
|
||||
#[error("no deployment of `{model}` is available")]
|
||||
NoHealthyDeployments { model: String },
|
||||
#[error("the attempt failed")]
|
||||
Attempt(#[from] AttemptError),
|
||||
#[error("the routing cache failed")]
|
||||
Cache(#[from] litellm_cache::Error),
|
||||
#[error("the router was retired")]
|
||||
Retired,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
/// The failed attempt behind this error, when a deployment was called.
|
||||
pub fn attempt(&self) -> Option<&AttemptError> {
|
||||
match self {
|
||||
Self::Attempt(attempt) => Some(attempt),
|
||||
Self::Config(_)
|
||||
| Self::UnknownModel { .. }
|
||||
| Self::NoHealthyDeployments { .. }
|
||||
| Self::Cache(_)
|
||||
| Self::Retired => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The class of a failed attempt, as litellm's exception mapping names it.
|
||||
#[derive(
|
||||
Clone,
|
||||
Copy,
|
||||
Debug,
|
||||
PartialEq,
|
||||
Eq,
|
||||
Hash,
|
||||
strum::Display,
|
||||
strum::EnumString,
|
||||
strum::IntoStaticStr,
|
||||
strum::VariantArray,
|
||||
)]
|
||||
pub enum FailureKind {
|
||||
#[strum(serialize = "RateLimitError")]
|
||||
RateLimit,
|
||||
#[strum(serialize = "ContextWindowExceededError")]
|
||||
ContextWindowExceeded,
|
||||
#[strum(serialize = "ContentPolicyViolationError")]
|
||||
ContentPolicyViolation,
|
||||
#[strum(serialize = "AuthenticationError")]
|
||||
Authentication,
|
||||
#[strum(serialize = "PermissionDeniedError")]
|
||||
PermissionDenied,
|
||||
#[strum(serialize = "NotFoundError")]
|
||||
NotFound,
|
||||
#[strum(serialize = "BadRequestError")]
|
||||
BadRequest,
|
||||
#[strum(serialize = "Timeout")]
|
||||
Timeout,
|
||||
#[strum(serialize = "APIConnectionError")]
|
||||
ApiConnection,
|
||||
#[strum(serialize = "InternalServerError")]
|
||||
InternalServer,
|
||||
#[strum(serialize = "ServiceUnavailableError")]
|
||||
ServiceUnavailable,
|
||||
#[strum(serialize = "BadGatewayError")]
|
||||
BadGateway,
|
||||
/// A guardrail blocked the request: never retried, never a fallback.
|
||||
#[strum(serialize = "GuardrailIntervention")]
|
||||
GuardrailIntervention,
|
||||
/// litellm rejected the request before calling the deployment (`MalformedRequestFieldError`,
|
||||
/// `UnsupportedEndpointError`): never cools the deployment down.
|
||||
#[strum(serialize = "Declined")]
|
||||
Declined,
|
||||
/// The caller's own credential failed, not the deployment's.
|
||||
#[strum(serialize = "CallerCredentialError")]
|
||||
CallerCredential,
|
||||
#[strum(serialize = "APIError")]
|
||||
Other,
|
||||
}
|
||||
|
||||
/// An opaque value the host hands through the router unchanged, such as the original Python
|
||||
/// exception or response object.
|
||||
#[derive(Clone)]
|
||||
pub struct Payload(pub Arc<dyn Any + Send + Sync>);
|
||||
|
||||
impl fmt::Debug for Payload {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter.write_str("Payload(..)")
|
||||
}
|
||||
}
|
||||
|
||||
/// One failed attempt, as typed facts the retry, fallback and cooldown rules decide by.
|
||||
#[derive(Clone, Debug, thiserror::Error)]
|
||||
#[error("the deployment call failed with {kind}")]
|
||||
pub struct AttemptError {
|
||||
pub kind: FailureKind,
|
||||
pub status: Option<u16>,
|
||||
/// The provider's `retry-after` (or `retry-after-ms`) header.
|
||||
pub retry_after: Option<Duration>,
|
||||
/// A deployment-level `num_retries` the failure carries (Python's `exception.num_retries`).
|
||||
pub num_retries: Option<u32>,
|
||||
/// The provider's error text, kept only for the cooldown record Python writes
|
||||
/// (`exception_received`, masked past 50 characters).
|
||||
pub detail: Option<Arc<str>>,
|
||||
pub payload: Option<Payload>,
|
||||
}
|
||||
|
||||
impl AttemptError {
|
||||
pub fn new(kind: FailureKind, status: Option<u16>) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
status,
|
||||
retry_after: None,
|
||||
num_retries: None,
|
||||
detail: None,
|
||||
payload: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_retry_after(self, retry_after: Duration) -> Self {
|
||||
Self {
|
||||
retry_after: Some(retry_after),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
/// The status Python's cooldown rules read: the provider's, else 500.
|
||||
pub fn cooldown_status(&self) -> u16 {
|
||||
self.status.unwrap_or(500)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::rate_limit("RateLimitError", FailureKind::RateLimit)]
|
||||
#[case::context_window("ContextWindowExceededError", FailureKind::ContextWindowExceeded)]
|
||||
#[case::caller_credential("CallerCredentialError", FailureKind::CallerCredential)]
|
||||
#[case::other("APIError", FailureKind::Other)]
|
||||
fn failure_kind_round_trips_its_exception_name(#[case] wire: &str, #[case] kind: FailureKind) {
|
||||
assert_eq!(wire.parse::<FailureKind>().unwrap(), kind);
|
||||
assert_eq!(<&'static str>::from(kind), wire);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty("")]
|
||||
#[case::lowercase("ratelimiterror")]
|
||||
fn failure_kind_rejects_unknown_names(#[case] wire: &str) {
|
||||
assert!(wire.parse::<FailureKind>().is_err());
|
||||
}
|
||||
}
|
||||
75
litellm-rust/crates/router-core/src/executor.rs
Normal file
75
litellm-rust/crates/router-core/src/executor.rs
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
//! Executors: each attempt resolves one after its deployment is picked. An (op, provider) pair in
|
||||
//! the route-rule table with a registered [`NativeRoute`] runs in-process; anything else goes
|
||||
//! through `Host::invoke`. An attempt runs on exactly one executor and is never replayed on
|
||||
//! another: a native failure is that attempt's failure, and a retry selects afresh.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::{
|
||||
config::Op,
|
||||
error::{AttemptError, Unsupported},
|
||||
host::{Attempt, Output},
|
||||
};
|
||||
|
||||
/// The (op, provider) pairs that have a Rust route machine.
|
||||
pub const NATIVE_RULES: &[(Op, &str)] = &[(Op::Messages, "anthropic")];
|
||||
|
||||
/// A Rust-native route machine (an `inference-*` crate) adapted to an attempt. The embedder
|
||||
/// registers one per native rule; for Messages on Anthropic that is an adapter over
|
||||
/// `litellm_inference_messages::MessagesRoute::execute`.
|
||||
#[async_trait]
|
||||
pub trait NativeRoute: Send + Sync {
|
||||
async fn execute(&self, attempt: &Attempt) -> Result<Output, AttemptError>;
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum Executor {
|
||||
Native(Arc<dyn NativeRoute>),
|
||||
Host,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Executors {
|
||||
native: Vec<(Op, String, Arc<dyn NativeRoute>)>,
|
||||
host_invokes: bool,
|
||||
}
|
||||
|
||||
impl Executors {
|
||||
pub fn new(host_invokes: bool) -> Self {
|
||||
Self {
|
||||
native: Vec::new(),
|
||||
host_invokes,
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a native route for a rule in [`NATIVE_RULES`]. A pair outside the table stays
|
||||
/// on the host, so a route cannot silently take over an operation not yet ported.
|
||||
pub fn register(&mut self, op: Op, provider: &str, route: Arc<dyn NativeRoute>) -> bool {
|
||||
let ruled = NATIVE_RULES
|
||||
.iter()
|
||||
.any(|(rule_op, rule_provider)| *rule_op == op && *rule_provider == provider);
|
||||
if ruled {
|
||||
self.native.push((op, provider.to_owned(), route));
|
||||
}
|
||||
ruled
|
||||
}
|
||||
|
||||
pub fn resolve(&self, op: &Op, provider: &str) -> Result<Executor, Unsupported> {
|
||||
if let Some((_, _, route)) = self
|
||||
.native
|
||||
.iter()
|
||||
.find(|(rule_op, rule_provider, _)| rule_op == op && rule_provider == provider)
|
||||
{
|
||||
return Ok(Executor::Native(route.clone()));
|
||||
}
|
||||
if self.host_invokes {
|
||||
return Ok(Executor::Host);
|
||||
}
|
||||
Err(Unsupported::Executor {
|
||||
op: op.clone(),
|
||||
provider: provider.to_owned(),
|
||||
})
|
||||
}
|
||||
}
|
||||
475
litellm-rust/crates/router-core/src/health.rs
Normal file
475
litellm-rust/crates/router-core/src/health.rs
Normal file
|
|
@ -0,0 +1,475 @@
|
|||
//! [`Health`]: cooldowns, allowed-fails counters and per-minute usage, kept in the shared
|
||||
//! [`RouterCache`] under the Python router's keys, encodings and TTLs:
|
||||
//!
|
||||
//! | state | key | value | TTL |
|
||||
//! |---|---|---|---|
|
||||
//! | cooldown | `deployment:{id}:cooldown` | `{exception_received, status_code, timestamp, cooldown_time}` | cooldown time |
|
||||
//! | allowed fails | `deployment:{id}:allowed_fails[:{suffix}]` | int counter | cooldown time |
|
||||
//! | usage | `global_router:{id}:{model}:tpm:{HH-MM}` and `:rpm:` | int counter | 60s |
|
||||
//! | outcomes (local only) | `{id}:successes`, `{id}:fails` | int counter | 60s |
|
||||
//!
|
||||
//! From `router_utils/cooldown_cache.py`, `router_utils/cooldown_handlers.py`,
|
||||
//! `router_utils/router_callbacks/track_deployment_metrics.py`, `RouterCacheEnum` in
|
||||
//! `types/router.py` and `Router._increment_usage` / `deployment_callback_on_failure` in
|
||||
//! `router.py`.
|
||||
|
||||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::debug;
|
||||
|
||||
use crate::{
|
||||
cache::{InMemoryRouterCache, RouterCache},
|
||||
error::{AttemptError, Error, FailureKind},
|
||||
registry::Deployment,
|
||||
settings::Settings,
|
||||
};
|
||||
|
||||
/// `RoutingArgs.ttl`: per-minute usage keys expire after a minute.
|
||||
pub const USAGE_TTL: Duration = Duration::from_secs(60);
|
||||
/// `DEFAULT_FAILURE_THRESHOLD_PERCENT`.
|
||||
pub const FAILURE_THRESHOLD_PERCENT: f64 = 0.5;
|
||||
/// `DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS`.
|
||||
pub const FAILURE_THRESHOLD_MINIMUM_REQUESTS: u64 = 5;
|
||||
/// `SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD`.
|
||||
pub const SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: u64 = 1000;
|
||||
/// `exception_masker` in `CooldownCache`: the visible prefix of `exception_received`.
|
||||
const EXCEPTION_VISIBLE_PREFIX: usize = 50;
|
||||
|
||||
pub trait Clock: Send + Sync {
|
||||
fn now(&self) -> SystemTime;
|
||||
}
|
||||
|
||||
pub struct SystemClock;
|
||||
|
||||
impl Clock for SystemClock {
|
||||
fn now(&self) -> SystemTime {
|
||||
SystemTime::now()
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_seconds(time: SystemTime) -> f64 {
|
||||
time.duration_since(UNIX_EPOCH)
|
||||
.map(|elapsed| elapsed.as_secs_f64())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `get_utc_datetime().strftime("%H-%M")`.
|
||||
pub fn current_minute(time: SystemTime) -> String {
|
||||
let seconds = time
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|elapsed| elapsed.as_secs())
|
||||
.unwrap_or_default()
|
||||
% 86_400;
|
||||
format!("{:02}-{:02}", seconds / 3600, (seconds % 3600) / 60)
|
||||
}
|
||||
|
||||
pub fn cooldown_key(id: &str) -> String {
|
||||
format!("deployment:{id}:cooldown")
|
||||
}
|
||||
|
||||
pub fn allowed_fails_key(id: &str, suffix: Option<&str>) -> String {
|
||||
match suffix {
|
||||
Some(suffix) => format!("deployment:{id}:allowed_fails:{suffix}"),
|
||||
None => format!("deployment:{id}:allowed_fails"),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)]
|
||||
pub enum UsageMetric {
|
||||
#[strum(serialize = "tpm")]
|
||||
Tpm,
|
||||
#[strum(serialize = "rpm")]
|
||||
Rpm,
|
||||
}
|
||||
|
||||
/// `RouterCacheEnum.TPM` / `RPM`: `model` is the deployment's `litellm_params.model`.
|
||||
pub fn usage_key(id: &str, model: &str, metric: UsageMetric, minute: &str) -> String {
|
||||
format!("global_router:{id}:{model}:{metric}:{minute}")
|
||||
}
|
||||
|
||||
/// `CooldownCacheValue`.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CooldownValue {
|
||||
pub exception_received: String,
|
||||
pub status_code: String,
|
||||
pub timestamp: f64,
|
||||
/// Whole seconds are written as an int, as Python's default `cooldown_time` is, so the
|
||||
/// "Try again in N seconds" text reads the same.
|
||||
#[serde(serialize_with = "whole_as_int")]
|
||||
pub cooldown_time: f64,
|
||||
}
|
||||
|
||||
fn whole_as_int<S: serde::Serializer>(value: &f64, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
if value.fract() == 0.0 && value.abs() < i64::MAX as f64 {
|
||||
serializer.serialize_i64(*value as i64)
|
||||
} else {
|
||||
serializer.serialize_f64(*value)
|
||||
}
|
||||
}
|
||||
|
||||
/// `litellm.should_retry`.
|
||||
pub fn should_retry_status(status: u16) -> bool {
|
||||
matches!(status, 408 | 409 | 429) || status >= 500
|
||||
}
|
||||
|
||||
/// `_is_cooldown_required`: a 4xx cools down only for 429, 401, 402, 404 and 408, every other
|
||||
/// status does, and a connection error never does.
|
||||
pub fn is_cooldown_required(failure: &AttemptError) -> bool {
|
||||
if failure.kind == FailureKind::ApiConnection {
|
||||
return false;
|
||||
}
|
||||
match failure.cooldown_status() {
|
||||
status @ 400..500 => matches!(status, 429 | 401 | 402 | 408 | 404),
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// `SensitiveDataMasker(visible_prefix=50, visible_suffix=0, mask_short_values=False)`.
|
||||
fn mask_exception(text: &str) -> String {
|
||||
let length = text.chars().count();
|
||||
if length <= EXCEPTION_VISIBLE_PREFIX {
|
||||
return text.to_owned();
|
||||
}
|
||||
let mut masked: String = text.chars().take(EXCEPTION_VISIBLE_PREFIX).collect();
|
||||
masked.extend(std::iter::repeat_n('*', length - EXCEPTION_VISIBLE_PREFIX));
|
||||
masked
|
||||
}
|
||||
|
||||
/// What the cooldown rules need to know about the deployment's group.
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct GroupShape {
|
||||
/// The deployment is the only one in its model group.
|
||||
pub single_deployment: bool,
|
||||
/// The requested group has a fallback configured.
|
||||
pub has_fallback_group: bool,
|
||||
}
|
||||
|
||||
/// An outcome the router did not see itself (`Health.report`).
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Outcome {
|
||||
Success { total_tokens: u64 },
|
||||
Failure(AttemptError),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Health {
|
||||
cache: Arc<dyn RouterCache>,
|
||||
local: InMemoryRouterCache,
|
||||
clock: Arc<dyn Clock>,
|
||||
}
|
||||
|
||||
impl Health {
|
||||
pub fn new(cache: Arc<dyn RouterCache>, clock: Arc<dyn Clock>) -> Self {
|
||||
Self {
|
||||
cache,
|
||||
local: InMemoryRouterCache::default(),
|
||||
clock,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cache(&self) -> &Arc<dyn RouterCache> {
|
||||
&self.cache
|
||||
}
|
||||
|
||||
fn now(&self) -> f64 {
|
||||
unix_seconds(self.clock.now())
|
||||
}
|
||||
|
||||
/// The ids among `ids` with an unexpired cooldown (`async_get_active_cooldowns`).
|
||||
pub async fn active_cooldowns(
|
||||
&self,
|
||||
ids: &[String],
|
||||
) -> Result<Vec<(String, CooldownValue)>, Error> {
|
||||
let keys: Vec<String> = ids.iter().map(|id| cooldown_key(id)).collect();
|
||||
let values = self.cache.async_batch_get_cache(&keys).await?;
|
||||
let now = self.now();
|
||||
Ok(ids
|
||||
.iter()
|
||||
.zip(values)
|
||||
.filter_map(|(id, value)| {
|
||||
let value: CooldownValue = serde_json::from_value(value?).ok()?;
|
||||
(value.timestamp + value.cooldown_time - now > 0.0).then(|| (id.clone(), value))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// `add_deployment_to_cooldown`.
|
||||
pub async fn add_cooldown(
|
||||
&self,
|
||||
id: &str,
|
||||
failure: &AttemptError,
|
||||
cooldown: Duration,
|
||||
) -> Result<(), Error> {
|
||||
let value = CooldownValue {
|
||||
exception_received: mask_exception(failure.detail.as_deref().unwrap_or_default()),
|
||||
status_code: failure.cooldown_status().to_string(),
|
||||
timestamp: self.now(),
|
||||
cooldown_time: cooldown.as_secs_f64(),
|
||||
};
|
||||
let value = serde_json::to_value(value).unwrap_or_default();
|
||||
self.cache
|
||||
.async_set_cache(&cooldown_key(id), value, Some(cooldown))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn local_count(&self, key: &str) -> u64 {
|
||||
self.local
|
||||
.get(key)
|
||||
.and_then(|value| value.as_f64())
|
||||
.map_or(0, |count| count as u64)
|
||||
}
|
||||
|
||||
/// The per-minute (successes, fails) this worker saw.
|
||||
pub fn outcomes_this_minute(&self, id: &str) -> (u64, u64) {
|
||||
(
|
||||
self.local_count(&format!("{id}:successes")),
|
||||
self.local_count(&format!("{id}:fails")),
|
||||
)
|
||||
}
|
||||
|
||||
/// Cooldown duration precedence from `deployment_callback_on_failure`: the deployment's
|
||||
/// `cooldown_time`, then the provider's retry-after, then the router default.
|
||||
pub fn cooldown_duration(
|
||||
settings: &Settings,
|
||||
deployment: &Deployment,
|
||||
failure: &AttemptError,
|
||||
) -> Duration {
|
||||
Self::configured_cooldown(deployment)
|
||||
.or(failure.retry_after)
|
||||
.unwrap_or(settings.cooldown_time)
|
||||
}
|
||||
|
||||
/// The deployment's own `cooldown_time`: `model_info` first, then `litellm_params`
|
||||
/// (`_first_present(model_info, litellm_params, key="cooldown_time")`). It is both the
|
||||
/// cooldown and the TTL of the deployment's `allowed_fails:generic` counter.
|
||||
fn configured_cooldown(deployment: &Deployment) -> Option<Duration> {
|
||||
deployment
|
||||
.model_info
|
||||
.cooldown_time
|
||||
.or_else(|| {
|
||||
deployment
|
||||
.raw_params
|
||||
.get("cooldown_time")
|
||||
.and_then(|v| v.as_f64())
|
||||
})
|
||||
.filter(|seconds| *seconds >= 0.0)
|
||||
.and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
|
||||
}
|
||||
|
||||
/// Record a failed attempt and cool the deployment down when the rules say so. Returns the
|
||||
/// cooldown applied, if any.
|
||||
pub async fn record_failure(
|
||||
&self,
|
||||
settings: &Settings,
|
||||
deployment: &Deployment,
|
||||
shape: GroupShape,
|
||||
failure: &AttemptError,
|
||||
) -> Result<Option<Duration>, Error> {
|
||||
let cooldown = Self::cooldown_duration(settings, deployment, failure);
|
||||
self.local
|
||||
.increment(&format!("{}:fails", deployment.id), 1.0, Some(USAGE_TTL));
|
||||
if !Self::should_run_cooldown_logic(settings, deployment, failure, cooldown) {
|
||||
return Ok(None);
|
||||
}
|
||||
if !self
|
||||
.should_cooldown(settings, deployment, shape, failure)
|
||||
.await?
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
debug!(deployment_id = %deployment.id, cooldown_seconds = cooldown.as_secs_f64(), "cooling deployment down");
|
||||
self.add_cooldown(&deployment.id, failure, cooldown).await?;
|
||||
Ok(Some(cooldown))
|
||||
}
|
||||
|
||||
/// `_should_run_cooldown_logic`.
|
||||
fn should_run_cooldown_logic(
|
||||
settings: &Settings,
|
||||
deployment: &Deployment,
|
||||
failure: &AttemptError,
|
||||
cooldown: Duration,
|
||||
) -> bool {
|
||||
!matches!(
|
||||
failure.kind,
|
||||
FailureKind::Declined | FailureKind::CallerCredential
|
||||
) && !cooldown.is_zero()
|
||||
&& !settings.disable_cooldowns
|
||||
&& is_cooldown_required(failure)
|
||||
&& !settings
|
||||
.provider_default_deployment_ids
|
||||
.contains(&deployment.id)
|
||||
}
|
||||
|
||||
/// `_should_cooldown_deployment`.
|
||||
async fn should_cooldown(
|
||||
&self,
|
||||
settings: &Settings,
|
||||
deployment: &Deployment,
|
||||
shape: GroupShape,
|
||||
failure: &AttemptError,
|
||||
) -> Result<bool, Error> {
|
||||
let status = failure.cooldown_status();
|
||||
let policy_fails = settings.allowed_fails_for(failure.kind);
|
||||
let lone = shape.single_deployment && !shape.has_fallback_group;
|
||||
if shape.single_deployment && status == 402 && policy_fails.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(allowed) = deployment.model_info.allowed_fails {
|
||||
// A deployment-wide `allowed_fails` keeps the lone-deployment safety net
|
||||
// (`_should_cooldown_based_on_deployment_policy`).
|
||||
if lone {
|
||||
return Ok(false);
|
||||
}
|
||||
let ttl = Self::configured_cooldown(deployment).unwrap_or(settings.cooldown_time);
|
||||
return self
|
||||
.exceeds_allowed_fails(&deployment.id, Some("generic"), allowed, ttl)
|
||||
.await;
|
||||
}
|
||||
if settings.allowed_fails_policy.is_none() && settings.allowed_fails.is_none() {
|
||||
let (successes, fails) = self.outcomes_this_minute(&deployment.id);
|
||||
let total = successes + fails;
|
||||
let percent_fails = if total > 0 {
|
||||
fails as f64 / total as f64
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
return Ok((status == 429 && !shape.single_deployment)
|
||||
|| (percent_fails == 1.0 && total >= SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD)
|
||||
|| (percent_fails > FAILURE_THRESHOLD_PERCENT
|
||||
&& total >= FAILURE_THRESHOLD_MINIMUM_REQUESTS
|
||||
&& !shape.single_deployment)
|
||||
|| !should_retry_status(status));
|
||||
}
|
||||
if lone && policy_fails.is_none() {
|
||||
return Ok(false);
|
||||
}
|
||||
let allowed = policy_fails.unwrap_or_else(|| settings.effective_allowed_fails());
|
||||
self.exceeds_allowed_fails(&deployment.id, None, allowed, settings.cooldown_time)
|
||||
.await
|
||||
}
|
||||
|
||||
/// `should_cooldown_based_on_allowed_fails_policy`: count this failure, then compare.
|
||||
async fn exceeds_allowed_fails(
|
||||
&self,
|
||||
id: &str,
|
||||
suffix: Option<&str>,
|
||||
allowed: u32,
|
||||
ttl: Duration,
|
||||
) -> Result<bool, Error> {
|
||||
let fails = self
|
||||
.cache
|
||||
.async_increment(&allowed_fails_key(id, suffix), 1.0, Some(ttl))
|
||||
.await?;
|
||||
Ok(fails > f64::from(allowed))
|
||||
}
|
||||
|
||||
/// Count a success, and the tokens and request against the deployment's per-minute usage
|
||||
/// when it has limits to track.
|
||||
pub async fn record_success(
|
||||
&self,
|
||||
deployment: &Deployment,
|
||||
total_tokens: u64,
|
||||
) -> Result<(), Error> {
|
||||
self.local.increment(
|
||||
&format!("{}:successes", deployment.id),
|
||||
1.0,
|
||||
Some(USAGE_TTL),
|
||||
);
|
||||
if !deployment.tracks_usage() {
|
||||
return Ok(());
|
||||
}
|
||||
let minute = current_minute(self.clock.now());
|
||||
let model = &deployment.params.model;
|
||||
self.cache
|
||||
.async_increment(
|
||||
&usage_key(&deployment.id, model, UsageMetric::Tpm, &minute),
|
||||
total_tokens as f64,
|
||||
Some(USAGE_TTL),
|
||||
)
|
||||
.await?;
|
||||
self.cache
|
||||
.async_increment(
|
||||
&usage_key(&deployment.id, model, UsageMetric::Rpm, &minute),
|
||||
1.0,
|
||||
Some(USAGE_TTL),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// This minute's (tpm, rpm) for a deployment.
|
||||
pub async fn usage(&self, deployment: &Deployment) -> Result<(u64, u64), Error> {
|
||||
let minute = current_minute(self.clock.now());
|
||||
let model = &deployment.params.model;
|
||||
let keys = [UsageMetric::Tpm, UsageMetric::Rpm]
|
||||
.map(|metric| usage_key(&deployment.id, model, metric, &minute));
|
||||
let values = self.cache.async_batch_get_cache(&keys).await?;
|
||||
let count = |index: usize| {
|
||||
values
|
||||
.get(index)
|
||||
.cloned()
|
||||
.flatten()
|
||||
.and_then(|value| value.as_f64())
|
||||
.map_or(0, |count| count as u64)
|
||||
};
|
||||
Ok((count(0), count(1)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::rate_limit(FailureKind::RateLimit, Some(429), true)]
|
||||
#[case::auth(FailureKind::Authentication, Some(401), true)]
|
||||
#[case::payment(FailureKind::Other, Some(402), true)]
|
||||
#[case::not_found(FailureKind::NotFound, Some(404), true)]
|
||||
#[case::timeout(FailureKind::Timeout, Some(408), true)]
|
||||
#[case::bad_request(FailureKind::BadRequest, Some(400), false)]
|
||||
#[case::forbidden(FailureKind::PermissionDenied, Some(403), false)]
|
||||
#[case::server(FailureKind::InternalServer, Some(500), true)]
|
||||
#[case::no_status(FailureKind::Other, None, true)]
|
||||
#[case::connection(FailureKind::ApiConnection, Some(500), false)]
|
||||
fn cooldown_required_follows_python(
|
||||
#[case] kind: FailureKind,
|
||||
#[case] status: Option<u16>,
|
||||
#[case] expected: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
is_cooldown_required(&AttemptError::new(kind, status)),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::midnight(0, "00-00")]
|
||||
#[case::afternoon(13 * 3600 + 7 * 60 + 59, "13-07")]
|
||||
fn current_minute_is_utc_hours_and_minutes(#[case] seconds: u64, #[case] expected: &str) {
|
||||
let time = UNIX_EPOCH + Duration::from_secs(86_400 * 3 + seconds);
|
||||
assert_eq!(current_minute(time), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn keys_match_python() {
|
||||
assert_eq!(
|
||||
(
|
||||
cooldown_key("abc"),
|
||||
allowed_fails_key("abc", Some("generic")),
|
||||
usage_key("abc", "openai/gpt-4o", UsageMetric::Tpm, "09-05"),
|
||||
),
|
||||
(
|
||||
"deployment:abc:cooldown".to_owned(),
|
||||
"deployment:abc:allowed_fails:generic".to_owned(),
|
||||
"global_router:abc:openai/gpt-4o:tpm:09-05".to_owned(),
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
168
litellm-rust/crates/router-core/src/host.rs
Normal file
168
litellm-rust/crates/router-core/src/host.rs
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
//! [`Host`]: what the embedding process provides. `PythonHost` serves it from the existing Python
|
||||
//! code; `NativeHost` serves it in the Rust server.
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::{
|
||||
cache::RouterCache,
|
||||
config::{Op, RoutingStrategy},
|
||||
error::{AttemptError, Error, FailureKind, Payload},
|
||||
health::Health,
|
||||
registry::{Deployment, Registry},
|
||||
settings::Settings,
|
||||
};
|
||||
|
||||
/// One call into the router.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct RouterRequest {
|
||||
/// The requested name: a model group, alias, routing group, deployment id or wildcard match.
|
||||
pub model: String,
|
||||
/// The request params, for native executors and plugins that read them.
|
||||
pub body: Value,
|
||||
/// `user_api_key_team_id`.
|
||||
pub team_id: Option<String>,
|
||||
pub specific_deployment: bool,
|
||||
/// A per-request `num_retries`, which wins over the router's.
|
||||
pub num_retries: Option<u32>,
|
||||
pub disable_fallbacks: bool,
|
||||
/// `_router_weights`: model group to deployment id to weight.
|
||||
pub router_weights: Option<BTreeMap<String, BTreeMap<String, f64>>>,
|
||||
/// What the host needs to run the call itself, such as the Python kwargs.
|
||||
pub payload: Option<Payload>,
|
||||
}
|
||||
|
||||
/// One attempt against one deployment.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Attempt {
|
||||
pub op: Op,
|
||||
pub request: Arc<RouterRequest>,
|
||||
pub deployment: Arc<Deployment>,
|
||||
/// The name this attempt resolved: the requested name, or the fallback target.
|
||||
pub name: String,
|
||||
/// The model group being served, after fallbacks.
|
||||
pub model_group: String,
|
||||
/// The `order` level an order fallback targets in the same group.
|
||||
pub target_order: Option<i64>,
|
||||
/// Every deployment `name` resolved to for this caller, the picked one included.
|
||||
pub candidate_ids: Vec<String>,
|
||||
/// 0 for the first attempt in this group, then the retry number.
|
||||
pub retry: u32,
|
||||
pub fallback_depth: u32,
|
||||
/// How many fallbacks deep the walk is: 0 for the requested name.
|
||||
pub fallback_level: u32,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// What a successful attempt returns.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Output {
|
||||
pub value: Payload,
|
||||
/// Prompt plus completion tokens, counted against the deployment's per-minute usage.
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
/// What a plugin or strategy sees: never the router itself.
|
||||
pub struct RoutingContext<'a> {
|
||||
pub registry: &'a Registry,
|
||||
pub settings: &'a Settings,
|
||||
pub health: &'a Health,
|
||||
pub request: &'a RouterRequest,
|
||||
pub op: &'a Op,
|
||||
pub model_group: &'a str,
|
||||
}
|
||||
|
||||
/// Which selection step the host is asked to run.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum SelectStage {
|
||||
/// Pre-call checks and sub-routers registered as plugins: return the deployments that remain.
|
||||
Plugins,
|
||||
/// A strategy not yet ported: return the candidates with the pick first.
|
||||
Strategy(RoutingStrategy),
|
||||
}
|
||||
|
||||
/// Routing events, delivered through [`Host::observe`].
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum RouterEvent {
|
||||
Selected {
|
||||
model_group: String,
|
||||
deployment_id: String,
|
||||
},
|
||||
Succeeded {
|
||||
deployment_id: String,
|
||||
},
|
||||
AttemptFailed {
|
||||
deployment_id: String,
|
||||
kind: FailureKind,
|
||||
status: Option<u16>,
|
||||
},
|
||||
CooldownSet {
|
||||
deployment_id: String,
|
||||
cooldown: Duration,
|
||||
},
|
||||
Retry {
|
||||
model_group: String,
|
||||
retry: u32,
|
||||
delay: Duration,
|
||||
},
|
||||
Fallback {
|
||||
from: String,
|
||||
to: String,
|
||||
/// The order level of `to` a same-group hop walks to.
|
||||
order: Option<i64>,
|
||||
depth: u32,
|
||||
/// The fallback level `to` is tried at: 1 for the request's own fallbacks.
|
||||
level: u32,
|
||||
},
|
||||
/// A group's retry loop ran every retry and the last one failed: Python stamps
|
||||
/// `num_retries` and `max_retries` on that error.
|
||||
RetriesExhausted {
|
||||
model_group: String,
|
||||
attempted: u32,
|
||||
max: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait Host: Send + Sync {
|
||||
/// Run a non-native attempt.
|
||||
async fn invoke(&self, attempt: &Attempt) -> Result<Output, AttemptError>;
|
||||
|
||||
/// Run a selection stage the core does not implement.
|
||||
async fn select(
|
||||
&self,
|
||||
stage: &SelectStage,
|
||||
ctx: &RoutingContext<'_>,
|
||||
candidates: Vec<Arc<Deployment>>,
|
||||
) -> Result<Vec<Arc<Deployment>>, Error>;
|
||||
|
||||
/// Access and budget checks before a cross-group fallback target.
|
||||
async fn allow_fallback(&self, request: &RouterRequest, target: &str) -> bool;
|
||||
|
||||
async fn sleep(&self, duration: Duration);
|
||||
|
||||
fn observe(&self, event: &RouterEvent);
|
||||
|
||||
/// The shared store routing state lives in.
|
||||
fn cache(&self) -> Arc<dyn RouterCache>;
|
||||
|
||||
/// Whether [`Self::invoke`] can run an attempt. A `NativeHost` cannot, so every served
|
||||
/// operation needs a native executor.
|
||||
fn can_invoke(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether [`Self::select`] can run strategies and plugins.
|
||||
fn can_select(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether [`Self::invoke`] records each attempt's outcome itself (cooldowns, allowed-fails
|
||||
/// counters, usage), as the Python router's own callbacks do. The core then leaves the
|
||||
/// health store to the host for host-run attempts, so no failure is counted twice.
|
||||
fn records_outcomes(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
|
@ -61,3 +61,285 @@
|
|||
//! Routing strategies, pre-call checks and sub-routers (auto, complexity, quality, adaptive)
|
||||
//! register through `register_plugin` and see a `RoutingContext` (registry, settings, health and
|
||||
//! the current request), never the router itself.
|
||||
//!
|
||||
//! # Modules
|
||||
//! - [`registry`]: deployments and their indices, `deployment_id`, `ChangeSet`.
|
||||
//! - [`config`] / [`settings`]: static and dynamic config, and the frozen `Settings` view.
|
||||
//! - [`cache`] / [`health`]: the shared routing cache, cooldowns and usage.
|
||||
//! - [`select`]: resolve, filter chain, simple-shuffle.
|
||||
//! - [`call`]: retries and fallbacks.
|
||||
//! - [`host`] / [`executor`]: the `Host` seam and per-attempt executors.
|
||||
|
||||
pub mod cache;
|
||||
pub mod call;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod executor;
|
||||
pub mod health;
|
||||
pub mod host;
|
||||
pub mod registry;
|
||||
pub mod select;
|
||||
pub mod settings;
|
||||
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use parking_lot::Mutex;
|
||||
|
||||
pub use crate::{
|
||||
cache::{InMemoryRouterCache, RouterCache},
|
||||
call::RouterResponse,
|
||||
config::{Op, RouterConfig, RouterDynamicConfig, RoutingStrategy},
|
||||
error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported},
|
||||
executor::{Executor, Executors, NativeRoute},
|
||||
health::{Clock, Health, Outcome, SystemClock},
|
||||
host::{Attempt, Host, Output, RouterEvent, RouterRequest, RoutingContext, SelectStage},
|
||||
registry::{ChangeSet, Deployment, DeploymentSpec, Registry, RegistryVersion},
|
||||
select::{Randomness, SeededRandomness, ThreadRandomness},
|
||||
settings::Settings,
|
||||
};
|
||||
use crate::{health::GroupShape, registry::DeploymentFilter, select::SelectInputs};
|
||||
|
||||
/// The registry and settings one call reads, swapped atomically by `apply`.
|
||||
pub(crate) struct Snapshot {
|
||||
pub registry: Arc<Registry>,
|
||||
pub settings: Arc<Settings>,
|
||||
}
|
||||
|
||||
pub(crate) struct Core {
|
||||
host: Arc<dyn Host>,
|
||||
executors: Executors,
|
||||
rng: Arc<dyn Randomness>,
|
||||
health: Health,
|
||||
config: RouterConfig,
|
||||
snapshot: ArcSwap<Snapshot>,
|
||||
writer: Mutex<()>,
|
||||
retired: AtomicBool,
|
||||
}
|
||||
|
||||
impl Core {
|
||||
/// Reject what this host cannot serve, before any request.
|
||||
fn validate(&self, registry: &Registry, settings: &Settings) -> Result<(), ConfigError> {
|
||||
let can_select = self.host.can_select();
|
||||
let strategies = std::iter::once(&settings.routing_strategy).chain(
|
||||
registry
|
||||
.routing_groups()
|
||||
.iter()
|
||||
.map(|group| &group.routing_strategy),
|
||||
);
|
||||
for strategy in strategies {
|
||||
if !strategy.is_native() && !can_select {
|
||||
return Err(Unsupported::Strategy(strategy.clone()).into());
|
||||
}
|
||||
}
|
||||
if let Some(plugin) = settings.plugins.first()
|
||||
&& !can_select
|
||||
{
|
||||
return Err(Unsupported::Plugin(plugin.name.clone()).into());
|
||||
}
|
||||
for deployment in registry.deployments(&DeploymentFilter::default()) {
|
||||
for op in &self.config.served_ops {
|
||||
self.executors.resolve(op, deployment.provider())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a [`Router`], with native routes and test seams.
|
||||
pub struct RouterBuilder {
|
||||
config: RouterConfig,
|
||||
dynamic: RouterDynamicConfig,
|
||||
host: Arc<dyn Host>,
|
||||
executors: Executors,
|
||||
rng: Arc<dyn Randomness>,
|
||||
clock: Arc<dyn Clock>,
|
||||
}
|
||||
|
||||
impl RouterBuilder {
|
||||
/// Register a native route for an (op, provider) rule; a pair outside
|
||||
/// [`executor::NATIVE_RULES`] is ignored and stays on the host.
|
||||
#[must_use]
|
||||
pub fn native_route(mut self, op: Op, provider: &str, route: Arc<dyn NativeRoute>) -> Self {
|
||||
self.executors.register(op, provider, route);
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn randomness(self, rng: Arc<dyn Randomness>) -> Self {
|
||||
Self { rng, ..self }
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn clock(self, clock: Arc<dyn Clock>) -> Self {
|
||||
Self { clock, ..self }
|
||||
}
|
||||
|
||||
pub fn build(self) -> Result<Router, ConfigError> {
|
||||
let registry = Registry::build(
|
||||
self.config.model_list.clone(),
|
||||
self.dynamic.model_alias_map.clone(),
|
||||
self.config.routing_groups.clone(),
|
||||
)?;
|
||||
let settings = Settings::new(&self.config, Arc::new(self.dynamic));
|
||||
let core = Core {
|
||||
health: Health::new(self.host.cache(), self.clock),
|
||||
host: self.host,
|
||||
executors: self.executors,
|
||||
rng: self.rng,
|
||||
config: self.config,
|
||||
snapshot: ArcSwap::from_pointee(Snapshot {
|
||||
registry: Arc::new(Registry::default()),
|
||||
settings: Arc::new(settings.clone()),
|
||||
}),
|
||||
writer: Mutex::new(()),
|
||||
retired: AtomicBool::new(false),
|
||||
};
|
||||
core.validate(®istry, &settings)?;
|
||||
core.snapshot.store(Arc::new(Snapshot {
|
||||
registry: Arc::new(registry),
|
||||
settings: Arc::new(settings),
|
||||
}));
|
||||
Ok(Router {
|
||||
core: Arc::new(core),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The router: the entry points `litellm.Router` delegates to.
|
||||
#[derive(Clone)]
|
||||
pub struct Router {
|
||||
core: Arc<Core>,
|
||||
}
|
||||
|
||||
impl Router {
|
||||
pub fn new(
|
||||
config: RouterConfig,
|
||||
dynamic: RouterDynamicConfig,
|
||||
host: Arc<dyn Host>,
|
||||
) -> Result<Self, ConfigError> {
|
||||
Self::builder(config, dynamic, host).build()
|
||||
}
|
||||
|
||||
pub fn builder(
|
||||
config: RouterConfig,
|
||||
dynamic: RouterDynamicConfig,
|
||||
host: Arc<dyn Host>,
|
||||
) -> RouterBuilder {
|
||||
RouterBuilder {
|
||||
executors: Executors::new(host.can_invoke()),
|
||||
config,
|
||||
dynamic,
|
||||
host,
|
||||
rng: Arc::new(ThreadRandomness),
|
||||
clock: Arc::new(SystemClock),
|
||||
}
|
||||
}
|
||||
|
||||
fn live(&self) -> Result<(), Error> {
|
||||
if self.core.retired.load(Ordering::Acquire) {
|
||||
return Err(Error::Retired);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Select, run the attempt, and retry or fall back until it succeeds or the policy is
|
||||
/// exhausted.
|
||||
pub async fn call(&self, op: Op, request: RouterRequest) -> Result<RouterResponse, Error> {
|
||||
self.live()?;
|
||||
self.core.call(op, Arc::new(request)).await
|
||||
}
|
||||
|
||||
/// Resolve, filter and pick a deployment without calling it.
|
||||
pub async fn select(&self, op: &Op, request: &RouterRequest) -> Result<Arc<Deployment>, Error> {
|
||||
self.live()?;
|
||||
let snapshot = self.core.snapshot.load_full();
|
||||
let candidates = self
|
||||
.core
|
||||
.candidates(
|
||||
&snapshot,
|
||||
op,
|
||||
request,
|
||||
&request.model,
|
||||
&SelectInputs::default(),
|
||||
)
|
||||
.await?;
|
||||
self.core.pick(&snapshot, op, request, &candidates).await
|
||||
}
|
||||
|
||||
pub fn registry(&self) -> Arc<Registry> {
|
||||
self.core.snapshot.load().registry.clone()
|
||||
}
|
||||
|
||||
pub fn settings(&self) -> Arc<Settings> {
|
||||
self.core.snapshot.load().settings.clone()
|
||||
}
|
||||
|
||||
pub fn health(&self) -> &Health {
|
||||
&self.core.health
|
||||
}
|
||||
|
||||
/// Record an outcome the router did not see itself, such as a health check.
|
||||
pub async fn report(&self, id: &str, outcome: Outcome) -> Result<(), Error> {
|
||||
let snapshot = self.core.snapshot.load_full();
|
||||
let Some(deployment) = snapshot.registry.deployment(id) else {
|
||||
return Ok(());
|
||||
};
|
||||
match outcome {
|
||||
Outcome::Success { total_tokens } => {
|
||||
self.core
|
||||
.health
|
||||
.record_success(&deployment, total_tokens)
|
||||
.await
|
||||
}
|
||||
Outcome::Failure(failure) => {
|
||||
let shape = GroupShape {
|
||||
single_deployment: snapshot
|
||||
.registry
|
||||
.group(&deployment.model_name)
|
||||
.is_some_and(|group| group.len() == 1),
|
||||
has_fallback_group: snapshot
|
||||
.settings
|
||||
.dynamic
|
||||
.fallback_targets(&[deployment.model_name.as_str()])
|
||||
.is_some(),
|
||||
};
|
||||
self.core
|
||||
.health
|
||||
.record_failure(&snapshot.settings, &deployment, shape, &failure)
|
||||
.await
|
||||
.map(drop)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply one change set atomically and return the new registry version.
|
||||
pub fn apply(&self, changes: ChangeSet) -> Result<RegistryVersion, ConfigError> {
|
||||
let _writer = self.core.writer.lock();
|
||||
let current = self.core.snapshot.load_full();
|
||||
let aliases = changes
|
||||
.settings
|
||||
.as_ref()
|
||||
.map(|dynamic| dynamic.model_alias_map.clone());
|
||||
let registry = current.registry.apply(&changes, aliases)?;
|
||||
let settings = match changes.settings {
|
||||
Some(dynamic) => Settings::new(&self.core.config, Arc::new(dynamic)),
|
||||
None => Settings::clone(¤t.settings),
|
||||
};
|
||||
self.core.validate(®istry, &settings)?;
|
||||
let version = registry.version();
|
||||
self.core.snapshot.store(Arc::new(Snapshot {
|
||||
registry: Arc::new(registry),
|
||||
settings: Arc::new(settings),
|
||||
}));
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
/// Stop serving: every later call and selection fails with [`Error::Retired`].
|
||||
pub fn retire(&self) {
|
||||
self.core.retired.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
820
litellm-rust/crates/router-core/src/registry.rs
Normal file
820
litellm-rust/crates/router-core/src/registry.rs
Normal file
|
|
@ -0,0 +1,820 @@
|
|||
//! The configured deployments, indexed by id, model group, alias, wildcard pattern, routing
|
||||
//! group and team. A [`Registry`] is immutable; [`Registry::apply`] builds the next version.
|
||||
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, HashMap},
|
||||
fmt::Write as _,
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use litellm_router_types::{LitellmParams, Spelled};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::error::ConfigError;
|
||||
|
||||
/// A registry's version; every change produces a higher one.
|
||||
pub type RegistryVersion = u64;
|
||||
|
||||
/// A deployment as a config or an `apply` spells it.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
pub struct DeploymentSpec {
|
||||
pub model_name: String,
|
||||
pub litellm_params: Map<String, Value>,
|
||||
#[serde(default)]
|
||||
pub model_info: Map<String, Value>,
|
||||
}
|
||||
|
||||
/// `routing_groups`: a named subset of model groups sharing a routing strategy.
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize)]
|
||||
pub struct RoutingGroupSpec {
|
||||
pub group_name: String,
|
||||
pub models: Vec<String>,
|
||||
pub routing_strategy: crate::config::RoutingStrategy,
|
||||
}
|
||||
|
||||
/// The `model_info` fields the router reads; every other key stays in [`Self::raw`].
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct ModelInfo {
|
||||
pub team_id: Option<String>,
|
||||
pub team_public_model_name: Option<String>,
|
||||
pub access_groups: Vec<String>,
|
||||
pub blocked: bool,
|
||||
pub cooldown_time: Option<f64>,
|
||||
pub allowed_fails: Option<u32>,
|
||||
pub raw: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl ModelInfo {
|
||||
fn from_raw(raw: Map<String, Value>) -> Self {
|
||||
let text = |key: &str| raw.get(key).and_then(Value::as_str).map(str::to_owned);
|
||||
Self {
|
||||
team_id: text("team_id"),
|
||||
team_public_model_name: text("team_public_model_name"),
|
||||
access_groups: raw
|
||||
.get("access_groups")
|
||||
.and_then(Value::as_array)
|
||||
.map(|groups| {
|
||||
groups
|
||||
.iter()
|
||||
.filter_map(Value::as_str)
|
||||
.map(str::to_owned)
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default(),
|
||||
blocked: raw.get("blocked").and_then(Value::as_bool) == Some(true),
|
||||
cooldown_time: raw.get("cooldown_time").and_then(Value::as_f64),
|
||||
allowed_fails: raw
|
||||
.get("allowed_fails")
|
||||
.and_then(Value::as_u64)
|
||||
.and_then(|fails| u32::try_from(fails).ok()),
|
||||
raw,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One configured deployment.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Deployment {
|
||||
pub id: String,
|
||||
pub model_name: String,
|
||||
pub params: LitellmParams,
|
||||
/// `litellm_params` as written, in order: what the id hashes and the host receives.
|
||||
pub raw_params: Map<String, Value>,
|
||||
pub model_info: ModelInfo,
|
||||
}
|
||||
|
||||
impl Deployment {
|
||||
pub fn from_spec(spec: DeploymentSpec) -> Result<Self, ConfigError> {
|
||||
let params: LitellmParams = serde_json::from_value(Value::Object(
|
||||
spec.litellm_params.clone(),
|
||||
))
|
||||
.map_err(|source| ConfigError::InvalidDeployment {
|
||||
model_name: spec.model_name.clone(),
|
||||
source,
|
||||
})?;
|
||||
let id = match spec.model_info.get("id") {
|
||||
Some(Value::String(id)) => id.clone(),
|
||||
Some(Value::Number(id)) => id.to_string(),
|
||||
_ => deployment_id(&spec.model_name, &spec.litellm_params),
|
||||
};
|
||||
Ok(Self {
|
||||
id,
|
||||
model_name: spec.model_name,
|
||||
params,
|
||||
raw_params: spec.litellm_params,
|
||||
model_info: ModelInfo::from_raw(spec.model_info),
|
||||
})
|
||||
}
|
||||
|
||||
/// The provider: `custom_llm_provider`, else the `model` prefix before `/`.
|
||||
pub fn provider(&self) -> &str {
|
||||
self.params
|
||||
.custom_llm_provider
|
||||
.as_deref()
|
||||
.or_else(|| {
|
||||
self.params
|
||||
.model
|
||||
.split_once('/')
|
||||
.map(|(provider, _)| provider)
|
||||
})
|
||||
.unwrap_or("openai")
|
||||
}
|
||||
|
||||
/// A numeric `litellm_params` value, as `simple_shuffle._metric_weight` reads it.
|
||||
pub fn metric(&self, key: &str) -> Option<f64> {
|
||||
let spelled = |value: &Option<Spelled<f64>>| match value {
|
||||
Some(Spelled::Value(value)) => Some(*value),
|
||||
Some(Spelled::Text(_)) | None => None,
|
||||
};
|
||||
match key {
|
||||
"rpm" => spelled(&self.params.rpm),
|
||||
"tpm" => spelled(&self.params.tpm),
|
||||
_ => self.raw_params.get(key).and_then(Value::as_f64),
|
||||
}
|
||||
}
|
||||
|
||||
/// `litellm_params.order` (`get_deployment_order`).
|
||||
pub fn order(&self) -> Option<i64> {
|
||||
self.raw_params.get("order").and_then(Value::as_i64)
|
||||
}
|
||||
|
||||
/// Whether `team_id` may use this deployment (`deployment_usable_by_team`).
|
||||
pub fn usable_by_team(&self, team_id: Option<&str>) -> bool {
|
||||
self.model_info
|
||||
.team_id
|
||||
.as_deref()
|
||||
.is_none_or(|owner| Some(owner) == team_id)
|
||||
}
|
||||
|
||||
/// Whether the router counts per-minute usage for this deployment
|
||||
/// (`_deployment_tracks_usage`, without the model-catalog limits).
|
||||
pub fn tracks_usage(&self) -> bool {
|
||||
self.params.tpm.is_some()
|
||||
|| self.params.rpm.is_some()
|
||||
|| self.params.itpm.is_some()
|
||||
|| self.params.otpm.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// The connection params of one deployment (`credentials(id)`).
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Credentials {
|
||||
pub provider: String,
|
||||
pub api_base: Option<String>,
|
||||
pub api_version: Option<String>,
|
||||
pub credential_name: Option<String>,
|
||||
pub params: LitellmParams,
|
||||
}
|
||||
|
||||
/// What a name resolved through.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ResolveKind {
|
||||
SpecificDeployment,
|
||||
Id,
|
||||
Alias,
|
||||
RoutingGroup,
|
||||
Group,
|
||||
Wildcard,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Resolution {
|
||||
pub kind: ResolveKind,
|
||||
/// The model group the deployments are routed as.
|
||||
pub group: String,
|
||||
pub deployments: Vec<Arc<Deployment>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub struct ResolveScope {
|
||||
/// The caller named a deployment's `litellm_params.model` (`specific_deployment=True`).
|
||||
pub specific_deployment: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct DeploymentFilter<'a> {
|
||||
pub model_name: Option<&'a str>,
|
||||
pub team_id: Option<&'a str>,
|
||||
pub provider: Option<&'a str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct NameFilter {
|
||||
pub include_aliases: bool,
|
||||
}
|
||||
|
||||
/// One atomic change: upserts, deletes and, optionally, a new dynamic config.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct ChangeSet {
|
||||
pub upserts: Vec<DeploymentSpec>,
|
||||
pub deletes: Vec<String>,
|
||||
pub settings: Option<crate::config::RouterDynamicConfig>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Registry {
|
||||
version: RegistryVersion,
|
||||
deployments: Vec<Arc<Deployment>>,
|
||||
by_id: HashMap<String, usize>,
|
||||
by_group: BTreeMap<String, Vec<usize>>,
|
||||
by_team: HashMap<String, Vec<usize>>,
|
||||
wildcards: Vec<(String, Vec<usize>)>,
|
||||
aliases: BTreeMap<String, String>,
|
||||
routing_groups: Vec<RoutingGroupSpec>,
|
||||
}
|
||||
|
||||
impl Registry {
|
||||
pub fn build(
|
||||
specs: Vec<DeploymentSpec>,
|
||||
aliases: BTreeMap<String, String>,
|
||||
routing_groups: Vec<RoutingGroupSpec>,
|
||||
) -> Result<Self, ConfigError> {
|
||||
let deployments = specs
|
||||
.into_iter()
|
||||
.map(|spec| Deployment::from_spec(spec).map(Arc::new))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let mut seen = BTreeSet::new();
|
||||
if let Some(duplicate) = deployments.iter().find(|d| !seen.insert(d.id.as_str())) {
|
||||
return Err(ConfigError::DuplicateId {
|
||||
id: duplicate.id.clone(),
|
||||
});
|
||||
}
|
||||
Ok(Self::index(0, deployments, aliases, routing_groups))
|
||||
}
|
||||
|
||||
fn index(
|
||||
version: RegistryVersion,
|
||||
deployments: Vec<Arc<Deployment>>,
|
||||
aliases: BTreeMap<String, String>,
|
||||
routing_groups: Vec<RoutingGroupSpec>,
|
||||
) -> Self {
|
||||
let mut by_id = HashMap::new();
|
||||
let mut by_group: BTreeMap<String, Vec<usize>> = BTreeMap::new();
|
||||
let mut by_team: HashMap<String, Vec<usize>> = HashMap::new();
|
||||
for (index, deployment) in deployments.iter().enumerate() {
|
||||
by_id.insert(deployment.id.clone(), index);
|
||||
by_group
|
||||
.entry(deployment.model_name.clone())
|
||||
.or_default()
|
||||
.push(index);
|
||||
if let Some(team) = &deployment.model_info.team_id {
|
||||
by_team.entry(team.clone()).or_default().push(index);
|
||||
}
|
||||
}
|
||||
// In the order each pattern first appears, then most specific first, as
|
||||
// `PatternUtils.sorted_patterns` keeps them: a stable sort, longest regex first,
|
||||
// then the most regex syntax.
|
||||
let mut wildcards: Vec<(String, Vec<usize>)> = Vec::new();
|
||||
for deployment in &deployments {
|
||||
let name = &deployment.model_name;
|
||||
if name.contains('*') && !wildcards.iter().any(|(pattern, _)| pattern == name) {
|
||||
wildcards.push((
|
||||
name.clone(),
|
||||
by_group.get(name).cloned().unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
}
|
||||
wildcards.sort_by_key(|(pattern, _)| std::cmp::Reverse(pattern_specificity(pattern)));
|
||||
Self {
|
||||
version,
|
||||
deployments,
|
||||
by_id,
|
||||
by_group,
|
||||
by_team,
|
||||
wildcards,
|
||||
aliases,
|
||||
routing_groups,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn version(&self) -> RegistryVersion {
|
||||
self.version
|
||||
}
|
||||
|
||||
/// The next registry: deletes, then upserts. An upserted deployment whose id exists is
|
||||
/// removed and appended at the end, as the Python router does on an edit.
|
||||
pub fn apply(
|
||||
&self,
|
||||
changes: &ChangeSet,
|
||||
aliases: Option<BTreeMap<String, String>>,
|
||||
) -> Result<Self, ConfigError> {
|
||||
let mut deployments = self.deployments.clone();
|
||||
deployments.retain(|deployment| !changes.deletes.contains(&deployment.id));
|
||||
for spec in &changes.upserts {
|
||||
let deployment = Deployment::from_spec(spec.clone())?;
|
||||
deployments.retain(|existing| existing.id != deployment.id);
|
||||
deployments.push(Arc::new(deployment));
|
||||
}
|
||||
Ok(Self::index(
|
||||
self.version + 1,
|
||||
deployments,
|
||||
aliases.unwrap_or_else(|| self.aliases.clone()),
|
||||
self.routing_groups.clone(),
|
||||
))
|
||||
}
|
||||
|
||||
fn at(&self, indices: &[usize]) -> Vec<Arc<Deployment>> {
|
||||
indices
|
||||
.iter()
|
||||
.map(|index| self.deployments[*index].clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn deployment(&self, id: &str) -> Option<Arc<Deployment>> {
|
||||
self.by_id
|
||||
.get(id)
|
||||
.map(|index| self.deployments[*index].clone())
|
||||
}
|
||||
|
||||
pub fn deployments(&self, filter: &DeploymentFilter<'_>) -> Vec<Arc<Deployment>> {
|
||||
let candidates = match (filter.model_name, filter.team_id) {
|
||||
(Some(name), _) => self
|
||||
.by_group
|
||||
.get(name)
|
||||
.map(|i| self.at(i))
|
||||
.unwrap_or_default(),
|
||||
(None, Some(team)) => self
|
||||
.by_team
|
||||
.get(team)
|
||||
.map(|i| self.at(i))
|
||||
.unwrap_or_default(),
|
||||
(None, None) => self.deployments.clone(),
|
||||
};
|
||||
candidates
|
||||
.into_iter()
|
||||
.filter(|d| {
|
||||
filter
|
||||
.team_id
|
||||
.is_none_or(|team| d.model_info.team_id.as_deref() == Some(team))
|
||||
})
|
||||
.filter(|d| {
|
||||
filter
|
||||
.provider
|
||||
.is_none_or(|provider| d.provider() == provider)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The model group names, in name order, with aliases when asked.
|
||||
pub fn names(&self, filter: &NameFilter) -> Vec<String> {
|
||||
let mut names: Vec<String> = self.by_group.keys().cloned().collect();
|
||||
if filter.include_aliases {
|
||||
names.extend(self.aliases.keys().cloned());
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
pub fn group(&self, name: &str) -> Option<Vec<Arc<Deployment>>> {
|
||||
self.by_group.get(name).map(|indices| self.at(indices))
|
||||
}
|
||||
|
||||
/// The routing group a model group belongs to, if any.
|
||||
pub fn routing_group_of(&self, model_name: &str) -> Option<&RoutingGroupSpec> {
|
||||
self.routing_groups
|
||||
.iter()
|
||||
.find(|group| group.models.iter().any(|model| model == model_name))
|
||||
}
|
||||
|
||||
/// The strategy a routing group named `group_name` declares.
|
||||
pub fn resolve_routing_group_strategy(
|
||||
&self,
|
||||
group_name: &str,
|
||||
) -> Option<crate::config::RoutingStrategy> {
|
||||
self.routing_groups
|
||||
.iter()
|
||||
.find(|group| group.group_name == group_name)
|
||||
.map(|group| group.routing_strategy.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn routing_groups(&self) -> &[RoutingGroupSpec] {
|
||||
&self.routing_groups
|
||||
}
|
||||
|
||||
/// Resolve a requested name, in the order of `_common_checks_available_deployment`:
|
||||
/// a specific deployment, an id, an alias, a routing group, a model group, a wildcard.
|
||||
pub fn resolve(&self, name: &str, scope: ResolveScope) -> Option<Resolution> {
|
||||
let resolution = |kind, group: &str, deployments: Vec<Arc<Deployment>>| {
|
||||
(!deployments.is_empty()).then(|| Resolution {
|
||||
kind,
|
||||
group: group.to_owned(),
|
||||
deployments,
|
||||
})
|
||||
};
|
||||
if scope.specific_deployment {
|
||||
let matching = self
|
||||
.deployments
|
||||
.iter()
|
||||
.filter(|d| d.params.model == name)
|
||||
.cloned()
|
||||
.collect();
|
||||
return resolution(ResolveKind::SpecificDeployment, name, matching);
|
||||
}
|
||||
if !self.by_group.contains_key(name)
|
||||
&& let Some(deployment) = self.deployment(name)
|
||||
{
|
||||
let group = deployment.model_name.clone();
|
||||
return resolution(ResolveKind::Id, &group, vec![deployment]);
|
||||
}
|
||||
if let Some(target) = self.aliases.get(name) {
|
||||
return self
|
||||
.group(target)
|
||||
.and_then(|deployments| resolution(ResolveKind::Alias, target, deployments));
|
||||
}
|
||||
if let Some(routing_group) = self.routing_groups.iter().find(|g| g.group_name == name) {
|
||||
let deployments = routing_group
|
||||
.models
|
||||
.iter()
|
||||
.filter_map(|model| self.by_group.get(model))
|
||||
.flat_map(|indices| self.at(indices))
|
||||
.collect();
|
||||
return resolution(ResolveKind::RoutingGroup, name, deployments);
|
||||
}
|
||||
if let Some(indices) = self.by_group.get(name) {
|
||||
return resolution(ResolveKind::Group, name, self.at(indices));
|
||||
}
|
||||
self.wildcards
|
||||
.iter()
|
||||
.find(|(pattern, _)| wildcard_matches(pattern, name))
|
||||
.and_then(|(pattern, indices)| {
|
||||
resolution(ResolveKind::Wildcard, pattern, self.at(indices))
|
||||
})
|
||||
}
|
||||
|
||||
/// Access group name to the model groups in it.
|
||||
pub fn access_groups(&self) -> BTreeMap<String, BTreeSet<String>> {
|
||||
let mut groups: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
|
||||
for deployment in &self.deployments {
|
||||
for group in &deployment.model_info.access_groups {
|
||||
groups
|
||||
.entry(group.clone())
|
||||
.or_default()
|
||||
.insert(deployment.model_name.clone());
|
||||
}
|
||||
}
|
||||
groups
|
||||
}
|
||||
|
||||
pub fn credentials(&self, id: &str) -> Option<Credentials> {
|
||||
self.deployment(id).map(|deployment| Credentials {
|
||||
provider: deployment.provider().to_owned(),
|
||||
api_base: deployment.params.api_base.clone(),
|
||||
api_version: deployment.params.api_version.clone(),
|
||||
credential_name: deployment.params.litellm_credential_name.clone(),
|
||||
params: deployment.params.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// `PatternUtils.calculate_pattern_specificity` over the pattern's regex
|
||||
/// (`re.escape(pattern).replace(r"\*", "(.*)")`): its length, then how much regex syntax it has.
|
||||
fn pattern_specificity(pattern: &str) -> (usize, usize) {
|
||||
let mut regex = String::new();
|
||||
for c in pattern.chars() {
|
||||
match c {
|
||||
'*' => regex.push_str("(.*)"),
|
||||
'(' | ')' | '[' | ']' | '{' | '}' | '?' | '+' | '-' | '|' | '^' | '$' | '\\' | '.'
|
||||
| '&' | '~' | '#' | ' ' | '\t' | '\n' | '\r' | '\x0b' | '\x0c' => {
|
||||
regex.push('\\');
|
||||
regex.push(c);
|
||||
}
|
||||
_ => regex.push(c),
|
||||
}
|
||||
}
|
||||
let complexity = regex
|
||||
.chars()
|
||||
.filter(|c| matches!(c, '*' | '+' | '?' | '\\' | '^' | '$' | '|' | '(' | ')'))
|
||||
.count();
|
||||
(regex.chars().count(), complexity)
|
||||
}
|
||||
|
||||
/// `*` matches any run of characters, as the pattern router's `(.*)` does under `re.match`:
|
||||
/// anchored at the start only, so the name may run on past the pattern's last literal.
|
||||
fn wildcard_matches(pattern: &str, name: &str) -> bool {
|
||||
let mut parts = pattern.split('*');
|
||||
let first = parts.next().unwrap_or_default();
|
||||
let Some(mut rest) = name.strip_prefix(first) else {
|
||||
return false;
|
||||
};
|
||||
let parts: Vec<&str> = parts.collect();
|
||||
for part in parts {
|
||||
match rest.find(part) {
|
||||
Some(at) => rest = &rest[at + part.len()..],
|
||||
None => return false,
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// `deployment_id` in `litellm/router_api.py`: sha256 over the model group and every
|
||||
/// `litellm_params` key and value in order. A string is taken as is, a dict as `json.dumps`, and
|
||||
/// anything else as Python's `str()`.
|
||||
pub fn deployment_id(model_group: &str, litellm_params: &Map<String, Value>) -> String {
|
||||
let mut text = String::from(model_group);
|
||||
for (key, value) in litellm_params {
|
||||
text.push_str(key);
|
||||
match value {
|
||||
Value::String(value) => text.push_str(value),
|
||||
Value::Object(_) => json_dumps(value, &mut text),
|
||||
Value::Null | Value::Bool(_) | Value::Number(_) | Value::Array(_) => {
|
||||
python_str(value, &mut text);
|
||||
}
|
||||
}
|
||||
}
|
||||
let digest = Sha256::digest(text.as_bytes());
|
||||
let mut hex = String::with_capacity(64);
|
||||
for byte in digest {
|
||||
let _ = write!(hex, "{byte:02x}");
|
||||
}
|
||||
hex
|
||||
}
|
||||
|
||||
/// Python's `repr(float)`: the shortest round-trip digits, positional for exponents in
|
||||
/// `-4..16`, scientific otherwise.
|
||||
fn python_float(value: f64, out: &mut String) {
|
||||
if value.is_nan() {
|
||||
out.push_str("nan");
|
||||
return;
|
||||
}
|
||||
if value.is_infinite() {
|
||||
out.push_str(if value > 0.0 { "inf" } else { "-inf" });
|
||||
return;
|
||||
}
|
||||
let scientific = format!("{value:e}");
|
||||
let (mantissa, exponent) = scientific.split_once('e').unwrap_or((&scientific, "0"));
|
||||
let exponent: i32 = exponent.parse().unwrap_or(0);
|
||||
let (sign, mantissa) = mantissa
|
||||
.strip_prefix('-')
|
||||
.map_or(("", mantissa), |m| ("-", m));
|
||||
let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
|
||||
out.push_str(sign);
|
||||
if (-4..16).contains(&exponent) {
|
||||
if exponent < 0 {
|
||||
out.push_str("0.");
|
||||
out.extend(std::iter::repeat_n('0', (-exponent - 1) as usize));
|
||||
out.push_str(&digits);
|
||||
} else {
|
||||
let point = exponent as usize + 1;
|
||||
if digits.len() <= point {
|
||||
out.push_str(&digits);
|
||||
out.extend(std::iter::repeat_n('0', point - digits.len()));
|
||||
out.push_str(".0");
|
||||
} else {
|
||||
out.push_str(&digits[..point]);
|
||||
out.push('.');
|
||||
out.push_str(&digits[point..]);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.push_str(&digits[..1]);
|
||||
if digits.len() > 1 {
|
||||
out.push('.');
|
||||
out.push_str(&digits[1..]);
|
||||
}
|
||||
let _ = write!(
|
||||
out,
|
||||
"e{}{:02}",
|
||||
if exponent < 0 { '-' } else { '+' },
|
||||
exponent.abs()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn number(value: &serde_json::Number, out: &mut String) {
|
||||
match value.as_f64() {
|
||||
Some(float) if !value.is_i64() && !value.is_u64() => python_float(float, out),
|
||||
Some(_) | None => out.push_str(&value.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// `json.dumps` with its defaults: `", "` and `": "` separators and `ensure_ascii`.
|
||||
fn json_dumps(value: &Value, out: &mut String) {
|
||||
match value {
|
||||
Value::Null => out.push_str("null"),
|
||||
Value::Bool(flag) => out.push_str(if *flag { "true" } else { "false" }),
|
||||
Value::Number(value) => number(value, out),
|
||||
Value::String(text) => json_string(text, out),
|
||||
Value::Array(items) => {
|
||||
out.push('[');
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push_str(", ");
|
||||
}
|
||||
json_dumps(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
Value::Object(entries) => {
|
||||
out.push('{');
|
||||
for (index, (key, item)) in entries.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push_str(", ");
|
||||
}
|
||||
json_string(key, out);
|
||||
out.push_str(": ");
|
||||
json_dumps(item, out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn json_string(text: &str, out: &mut String) {
|
||||
out.push('"');
|
||||
for ch in text.chars() {
|
||||
match ch {
|
||||
'"' => out.push_str("\\\""),
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
'\u{08}' => out.push_str("\\b"),
|
||||
'\u{0c}' => out.push_str("\\f"),
|
||||
ch if ch.is_ascii() && !ch.is_ascii_control() => out.push(ch),
|
||||
ch => {
|
||||
let mut units = [0u16; 2];
|
||||
for unit in ch.encode_utf16(&mut units) {
|
||||
let _ = write!(out, "\\u{unit:04x}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
|
||||
/// Python's `str()` of a JSON value: `repr` for the items of a list or dict.
|
||||
fn python_str(value: &Value, out: &mut String) {
|
||||
match value {
|
||||
Value::Null => out.push_str("None"),
|
||||
Value::Bool(flag) => out.push_str(if *flag { "True" } else { "False" }),
|
||||
Value::Number(value) => number(value, out),
|
||||
Value::String(text) => python_repr_str(text, out),
|
||||
Value::Array(items) => {
|
||||
out.push('[');
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push_str(", ");
|
||||
}
|
||||
python_str(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
Value::Object(entries) => {
|
||||
out.push('{');
|
||||
for (index, (key, item)) in entries.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.push_str(", ");
|
||||
}
|
||||
python_repr_str(key, out);
|
||||
out.push_str(": ");
|
||||
python_str(item, out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn python_repr_str(text: &str, out: &mut String) {
|
||||
let quote = if text.contains('\'') && !text.contains('"') {
|
||||
'"'
|
||||
} else {
|
||||
'\''
|
||||
};
|
||||
out.push(quote);
|
||||
for ch in text.chars() {
|
||||
match ch {
|
||||
'\\' => out.push_str("\\\\"),
|
||||
'\n' => out.push_str("\\n"),
|
||||
'\r' => out.push_str("\\r"),
|
||||
'\t' => out.push_str("\\t"),
|
||||
ch if ch == quote => {
|
||||
out.push('\\');
|
||||
out.push(ch);
|
||||
}
|
||||
ch if (ch as u32) < 0x20 || ch as u32 == 0x7f => {
|
||||
let _ = write!(out, "\\x{:02x}", ch as u32);
|
||||
}
|
||||
ch => out.push(ch),
|
||||
}
|
||||
}
|
||||
out.push(quote);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn spec(model_name: &str, params: Value, info: Value) -> DeploymentSpec {
|
||||
serde_json::from_value(json!({
|
||||
"model_name": model_name, "litellm_params": params, "model_info": info,
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// Expected ids computed with `litellm.router_api.deployment_id`.
|
||||
#[rstest]
|
||||
#[case::strings(
|
||||
"g",
|
||||
json!({"model": "openai/gpt-4o", "api_key": "sk-1"}),
|
||||
"30256ba697cc98ce2aff32c3b73c4b90514cb199876aef429ffcd91ae826d025"
|
||||
)]
|
||||
#[case::scalars(
|
||||
"claude",
|
||||
json!({"model": "anthropic/claude-3", "rpm": 10, "timeout": 1.5, "stream": true, "x": null}),
|
||||
"645fd1504089c1084316573ccfc5dd1dd9622623b169026657484fb64a2c89a2"
|
||||
)]
|
||||
#[case::list_and_dict(
|
||||
"m",
|
||||
json!({"model": "azure/x", "tags": ["a", "b'c"], "extra_body": {"k": [1, 2.0, "é"], "z": {"q": false}}}),
|
||||
"64f11e16dabed321268bd92132de2020290ef193c0cf554b2c74faee8badbb96"
|
||||
)]
|
||||
fn deployment_id_matches_python(
|
||||
#[case] group: &str,
|
||||
#[case] params: Value,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let Value::Object(params) = params else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(deployment_id(group, ¶ms), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::integral(2.0, "2.0")]
|
||||
#[case::small(0.00001, "1e-05")]
|
||||
#[case::large(1e16, "1e+16")]
|
||||
#[case::fraction(0.1, "0.1")]
|
||||
fn floats_print_as_python_repr(#[case] value: f64, #[case] expected: &str) {
|
||||
let mut out = String::new();
|
||||
python_float(value, &mut out);
|
||||
assert_eq!(out, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::group("gpt-4", ResolveKind::Group, "gpt-4")]
|
||||
#[case::alias("smart", ResolveKind::Alias, "gpt-4")]
|
||||
#[case::id("dep-b", ResolveKind::Id, "gpt-4")]
|
||||
#[case::wildcard("anthropic/claude-x", ResolveKind::Wildcard, "anthropic/*")]
|
||||
fn resolve_follows_python_order(
|
||||
#[case] name: &str,
|
||||
#[case] kind: ResolveKind,
|
||||
#[case] group: &str,
|
||||
) {
|
||||
let registry = Registry::build(
|
||||
vec![
|
||||
spec(
|
||||
"gpt-4",
|
||||
json!({"model": "openai/gpt-4"}),
|
||||
json!({"id": "dep-a"}),
|
||||
),
|
||||
spec(
|
||||
"gpt-4",
|
||||
json!({"model": "azure/gpt-4"}),
|
||||
json!({"id": "dep-b"}),
|
||||
),
|
||||
spec("anthropic/*", json!({"model": "anthropic/*"}), json!({})),
|
||||
],
|
||||
BTreeMap::from([("smart".to_owned(), "gpt-4".to_owned())]),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let resolution = registry.resolve(name, ResolveScope::default()).unwrap();
|
||||
assert_eq!((resolution.kind, resolution.group.as_str()), (kind, group));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn apply_moves_an_edited_deployment_to_the_end() {
|
||||
let registry = Registry::build(
|
||||
vec![
|
||||
spec("a", json!({"model": "openai/a"}), json!({"id": "1"})),
|
||||
spec("a", json!({"model": "openai/b"}), json!({"id": "2"})),
|
||||
],
|
||||
BTreeMap::new(),
|
||||
Vec::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let next = registry
|
||||
.apply(
|
||||
&ChangeSet {
|
||||
upserts: vec![spec("a", json!({"model": "openai/c"}), json!({"id": "1"}))],
|
||||
..ChangeSet::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let ids: Vec<String> = next
|
||||
.group("a")
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|d| d.id.clone())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
(next.version(), ids),
|
||||
(1, vec!["2".to_owned(), "1".to_owned()])
|
||||
);
|
||||
}
|
||||
}
|
||||
320
litellm-rust/crates/router-core/src/select.rs
Normal file
320
litellm-rust/crates/router-core/src/select.rs
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
//! Selection: resolve a name, filter its deployments in the order of
|
||||
//! `Router._core_async_get_healthy_deployments` (team, cooldown, blocked, plugins, order, retry
|
||||
//! skips), then pick one with the strategy. Simple-shuffle is native
|
||||
//! (`router_strategy/simple_shuffle.py`); other strategies go through `Host::select`.
|
||||
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use rand::{Rng, SeedableRng, rngs::StdRng};
|
||||
|
||||
use crate::{
|
||||
Core, Snapshot,
|
||||
config::{Op, RoutingStrategy},
|
||||
error::Error,
|
||||
host::{RouterRequest, RoutingContext, SelectStage},
|
||||
registry::{Deployment, ResolveKind, ResolveScope},
|
||||
};
|
||||
|
||||
/// Randomness for the shuffle and retry jitter, injectable so tests are deterministic.
|
||||
pub trait Randomness: Send + Sync {
|
||||
/// A float in `[0, 1)`, as Python's `random.random()`.
|
||||
fn next_f64(&self) -> f64;
|
||||
}
|
||||
|
||||
pub struct ThreadRandomness;
|
||||
|
||||
impl Randomness for ThreadRandomness {
|
||||
fn next_f64(&self) -> f64 {
|
||||
rand::thread_rng().r#gen()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct SeededRandomness(Mutex<StdRng>);
|
||||
|
||||
impl SeededRandomness {
|
||||
pub fn new(seed: u64) -> Self {
|
||||
Self(Mutex::new(StdRng::seed_from_u64(seed)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Randomness for SeededRandomness {
|
||||
fn next_f64(&self) -> f64 {
|
||||
self.0.lock().r#gen()
|
||||
}
|
||||
}
|
||||
|
||||
/// A group's deployments before and after filtering.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Candidates {
|
||||
pub model_group: String,
|
||||
pub kind: ResolveKind,
|
||||
/// Every deployment the caller may use (after the team filter).
|
||||
pub all: Vec<Arc<Deployment>>,
|
||||
pub healthy: Vec<Arc<Deployment>>,
|
||||
}
|
||||
|
||||
/// Per-attempt selection inputs the call loop carries.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub(crate) struct SelectInputs {
|
||||
pub target_order: Option<i64>,
|
||||
pub skipped_ids: Vec<String>,
|
||||
/// Stop before the plugins, order and skip filters: the group less cooldowns and blocked
|
||||
/// deployments, which `_async_get_healthy_deployments` counts when deciding whether to retry.
|
||||
pub count_only: bool,
|
||||
}
|
||||
|
||||
/// `simple_shuffle`: weighted by `_router_weights[group]`, else `weight`, else `rpm`, else
|
||||
/// `tpm`, each normalized by its maximum, else uniform.
|
||||
pub fn simple_shuffle(
|
||||
deployments: &[Arc<Deployment>],
|
||||
model_group: &str,
|
||||
router_weights: Option<&BTreeMap<String, BTreeMap<String, f64>>>,
|
||||
rng: &dyn Randomness,
|
||||
) -> Option<Arc<Deployment>> {
|
||||
if deployments.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let scoped = router_weights
|
||||
.and_then(|weights| weights.get(model_group))
|
||||
.filter(|weights| !weights.is_empty())
|
||||
.map(|weights| {
|
||||
deployments
|
||||
.iter()
|
||||
.map(|d| weights.get(&d.id).copied().unwrap_or(0.0))
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
let metric_sets = ["weight", "rpm", "tpm"].map(|metric| {
|
||||
deployments
|
||||
.iter()
|
||||
.map(|d| d.metric(metric).unwrap_or(0.0))
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
for weights in scoped.into_iter().chain(metric_sets) {
|
||||
let largest = weights.iter().copied().fold(0.0, f64::max);
|
||||
if largest <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
let normalized: Vec<f64> = weights.iter().map(|weight| weight / largest).collect();
|
||||
let total: f64 = normalized.iter().sum();
|
||||
if total <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
// `random.choices`: bisect_right over the cumulative weights.
|
||||
let target = rng.next_f64() * total;
|
||||
let mut cumulative = 0.0;
|
||||
let index = normalized
|
||||
.iter()
|
||||
.position(|weight| {
|
||||
cumulative += weight;
|
||||
target < cumulative
|
||||
})
|
||||
.unwrap_or(deployments.len() - 1);
|
||||
return Some(deployments[index].clone());
|
||||
}
|
||||
let index = ((rng.next_f64() * deployments.len() as f64) as usize).min(deployments.len() - 1);
|
||||
Some(deployments[index].clone())
|
||||
}
|
||||
|
||||
impl Core {
|
||||
fn context<'a>(
|
||||
&'a self,
|
||||
snapshot: &'a Snapshot,
|
||||
op: &'a Op,
|
||||
request: &'a RouterRequest,
|
||||
model_group: &'a str,
|
||||
) -> RoutingContext<'a> {
|
||||
RoutingContext {
|
||||
registry: &snapshot.registry,
|
||||
settings: &snapshot.settings,
|
||||
health: &self.health,
|
||||
request,
|
||||
op,
|
||||
model_group,
|
||||
}
|
||||
}
|
||||
|
||||
/// The eligible deployments for `name`, in filter-chain order.
|
||||
pub(crate) async fn candidates(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
op: &Op,
|
||||
request: &RouterRequest,
|
||||
name: &str,
|
||||
inputs: &SelectInputs,
|
||||
) -> Result<Candidates, Error> {
|
||||
let registry = &snapshot.registry;
|
||||
let scope = ResolveScope {
|
||||
specific_deployment: request.specific_deployment,
|
||||
};
|
||||
let dynamic = &snapshot.settings.dynamic;
|
||||
let resolution = registry
|
||||
.resolve(name, scope)
|
||||
.or_else(|| {
|
||||
// No deployment for the name: route to the first default fallback.
|
||||
let default = dynamic
|
||||
.fallbacks
|
||||
.default_target()
|
||||
.or_else(|| dynamic.default_fallbacks.first().map(String::as_str))?;
|
||||
registry.resolve(default, scope)
|
||||
})
|
||||
.ok_or_else(|| Error::UnknownModel {
|
||||
model: name.to_owned(),
|
||||
})?;
|
||||
let team_id = request.team_id.as_deref();
|
||||
let all: Vec<Arc<Deployment>> = resolution
|
||||
.deployments
|
||||
.into_iter()
|
||||
.filter(|d| d.usable_by_team(team_id))
|
||||
.collect();
|
||||
|
||||
let ids: Vec<String> = all.iter().map(|d| d.id.clone()).collect();
|
||||
let cooling = self.health.active_cooldowns(&ids).await?;
|
||||
let mut healthy: Vec<Arc<Deployment>> = all
|
||||
.iter()
|
||||
.filter(|d| !cooling.iter().any(|(id, _)| *id == d.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
let settings = &snapshot.settings;
|
||||
if healthy.is_empty()
|
||||
&& settings.enable_health_check_routing
|
||||
&& settings.allowed_fails_policy.is_some()
|
||||
{
|
||||
healthy = all.clone();
|
||||
}
|
||||
healthy.retain(|d| !d.model_info.blocked);
|
||||
if inputs.count_only {
|
||||
return Ok(Candidates {
|
||||
model_group: resolution.group,
|
||||
kind: resolution.kind,
|
||||
all,
|
||||
healthy,
|
||||
});
|
||||
}
|
||||
|
||||
if !settings.plugins.is_empty() && !healthy.is_empty() {
|
||||
let ctx = self.context(snapshot, op, request, &resolution.group);
|
||||
healthy = self
|
||||
.host
|
||||
.select(&SelectStage::Plugins, &ctx, healthy)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let order = inputs
|
||||
.target_order
|
||||
.or_else(|| healthy.iter().filter_map(|d| d.order()).min());
|
||||
if let Some(order) = order {
|
||||
healthy.retain(|d| d.order() == Some(order));
|
||||
}
|
||||
|
||||
if !inputs.skipped_ids.is_empty() {
|
||||
let kept: Vec<Arc<Deployment>> = healthy
|
||||
.iter()
|
||||
.filter(|d| !inputs.skipped_ids.contains(&d.id))
|
||||
.cloned()
|
||||
.collect();
|
||||
if !kept.is_empty() {
|
||||
healthy = kept;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Candidates {
|
||||
model_group: resolution.group,
|
||||
kind: resolution.kind,
|
||||
all,
|
||||
healthy,
|
||||
})
|
||||
}
|
||||
|
||||
/// Pick one of `candidates.healthy` with the group's strategy.
|
||||
pub(crate) async fn pick(
|
||||
&self,
|
||||
snapshot: &Snapshot,
|
||||
op: &Op,
|
||||
request: &RouterRequest,
|
||||
candidates: &Candidates,
|
||||
) -> Result<Arc<Deployment>, Error> {
|
||||
let no_deployment = || Error::NoHealthyDeployments {
|
||||
model: candidates.model_group.clone(),
|
||||
};
|
||||
let strategy = match candidates.kind {
|
||||
ResolveKind::RoutingGroup => snapshot
|
||||
.registry
|
||||
.resolve_routing_group_strategy(&candidates.model_group),
|
||||
ResolveKind::SpecificDeployment
|
||||
| ResolveKind::Id
|
||||
| ResolveKind::Alias
|
||||
| ResolveKind::Group
|
||||
| ResolveKind::Wildcard => None,
|
||||
}
|
||||
.unwrap_or_else(|| snapshot.settings.routing_strategy.clone());
|
||||
if strategy == RoutingStrategy::SimpleShuffle {
|
||||
return simple_shuffle(
|
||||
&candidates.healthy,
|
||||
&candidates.model_group,
|
||||
request.router_weights.as_ref(),
|
||||
self.rng.as_ref(),
|
||||
)
|
||||
.ok_or_else(no_deployment);
|
||||
}
|
||||
let ctx = self.context(snapshot, op, request, &candidates.model_group);
|
||||
self.host
|
||||
.select(
|
||||
&SelectStage::Strategy(strategy),
|
||||
&ctx,
|
||||
candidates.healthy.clone(),
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.next()
|
||||
.ok_or_else(no_deployment)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::registry::DeploymentSpec;
|
||||
|
||||
struct Fixed(f64);
|
||||
|
||||
impl Randomness for Fixed {
|
||||
fn next_f64(&self) -> f64 {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn deployment(id: &str, params: serde_json::Value) -> Arc<Deployment> {
|
||||
let spec: DeploymentSpec = serde_json::from_value(json!({
|
||||
"model_name": "g", "litellm_params": params, "model_info": {"id": id},
|
||||
}))
|
||||
.unwrap();
|
||||
Arc::new(Deployment::from_spec(spec).unwrap())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::weight_low_draw(0.1, "a")]
|
||||
#[case::weight_high_draw(0.9, "b")]
|
||||
fn shuffle_uses_weight_before_rpm(#[case] draw: f64, #[case] expected: &str) {
|
||||
let deployments = [
|
||||
deployment("a", json!({"model": "openai/a", "weight": 1, "rpm": 1000})),
|
||||
deployment("b", json!({"model": "openai/b", "weight": 3})),
|
||||
];
|
||||
let picked = simple_shuffle(&deployments, "g", None, &Fixed(draw)).unwrap();
|
||||
assert_eq!(picked.id, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn shuffle_prefers_request_weights() {
|
||||
let deployments = [
|
||||
deployment("a", json!({"model": "openai/a", "weight": 100})),
|
||||
deployment("b", json!({"model": "openai/b"})),
|
||||
];
|
||||
let weights = BTreeMap::from([("g".to_owned(), BTreeMap::from([("b".to_owned(), 1.0)]))]);
|
||||
let picked = simple_shuffle(&deployments, "g", Some(&weights), &Fixed(0.99)).unwrap();
|
||||
assert_eq!(picked.id, "b");
|
||||
}
|
||||
}
|
||||
69
litellm-rust/crates/router-core/src/settings.rs
Normal file
69
litellm-rust/crates/router-core/src/settings.rs
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
//! [`Settings`]: one typed, frozen view of the configuration a caller may read, built from the
|
||||
//! static config and the current dynamic snapshot.
|
||||
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use crate::{
|
||||
config::{
|
||||
AllowedFailsPolicy, DEFAULT_COOLDOWN_TIME, PluginSpec, RetryPolicy, RouterConfig,
|
||||
RouterDynamicConfig, RoutingStrategy,
|
||||
},
|
||||
error::FailureKind,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Settings {
|
||||
pub routing_strategy: RoutingStrategy,
|
||||
pub plugins: Arc<[PluginSpec]>,
|
||||
pub cooldown_time: Duration,
|
||||
pub disable_cooldowns: bool,
|
||||
pub allowed_fails: Option<u32>,
|
||||
pub allowed_fails_policy: Option<AllowedFailsPolicy>,
|
||||
pub retry_policy: Option<RetryPolicy>,
|
||||
pub num_retries: u32,
|
||||
pub max_fallbacks: u32,
|
||||
pub retry_after: Duration,
|
||||
pub timeout: Option<Duration>,
|
||||
pub drop_params: Option<bool>,
|
||||
pub enable_health_check_routing: bool,
|
||||
pub provider_default_deployment_ids: Arc<[String]>,
|
||||
pub dynamic: Arc<RouterDynamicConfig>,
|
||||
}
|
||||
|
||||
fn seconds(value: f64) -> Duration {
|
||||
Duration::try_from_secs_f64(value).unwrap_or_default()
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn new(config: &RouterConfig, dynamic: Arc<RouterDynamicConfig>) -> Self {
|
||||
Self {
|
||||
routing_strategy: config.routing_strategy.clone(),
|
||||
plugins: config.plugins.clone().into(),
|
||||
cooldown_time: config.cooldown_time.map_or(DEFAULT_COOLDOWN_TIME, seconds),
|
||||
disable_cooldowns: config.disable_cooldowns,
|
||||
allowed_fails: dynamic.allowed_fails,
|
||||
allowed_fails_policy: config.allowed_fails_policy.clone(),
|
||||
retry_policy: config.retry_policy.clone(),
|
||||
num_retries: dynamic.num_retries.unwrap_or(0),
|
||||
max_fallbacks: dynamic.max_fallbacks,
|
||||
retry_after: seconds(config.retry_after),
|
||||
timeout: config.request_timeout.or(config.timeout).map(seconds),
|
||||
drop_params: config.drop_params,
|
||||
enable_health_check_routing: config.enable_health_check_routing,
|
||||
provider_default_deployment_ids: config.provider_default_deployment_ids.clone().into(),
|
||||
dynamic,
|
||||
}
|
||||
}
|
||||
|
||||
/// `allowed_fails_for`: the policy's count for this failure kind.
|
||||
pub fn allowed_fails_for(&self, kind: FailureKind) -> Option<u32> {
|
||||
self.allowed_fails_policy
|
||||
.as_ref()
|
||||
.and_then(|policy| policy.allowed_fails_for(kind))
|
||||
}
|
||||
|
||||
/// Python's `allowed_fails` default when set without a value (`litellm.allowed_fails`).
|
||||
pub fn effective_allowed_fails(&self) -> u32 {
|
||||
self.allowed_fails.unwrap_or(3)
|
||||
}
|
||||
}
|
||||
167
litellm-rust/crates/router-core/tests/call_loop.rs
Normal file
167
litellm-rust/crates/router-core/tests/call_loop.rs
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
//! A call through the router end to end: selection, retries, fallbacks and cooldowns against a
|
||||
//! scripted host.
|
||||
|
||||
use std::{collections::VecDeque, sync::Arc, time::Duration};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use litellm_router_core::{
|
||||
Attempt, AttemptError, Deployment, Error, FailureKind, Host, InMemoryRouterCache, Op, Output,
|
||||
Payload, Router, RouterCache, RouterConfig, RouterDynamicConfig, RouterEvent, RouterRequest,
|
||||
RoutingContext, SeededRandomness, SelectStage,
|
||||
};
|
||||
use parking_lot::Mutex;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
|
||||
#[derive(Default)]
|
||||
struct ScriptedHost {
|
||||
/// Failures handed out in order, keyed by model group; a group with none left succeeds.
|
||||
script: Mutex<Vec<(String, VecDeque<AttemptError>)>>,
|
||||
events: Mutex<Vec<RouterEvent>>,
|
||||
cache: InMemoryRouterCache,
|
||||
}
|
||||
|
||||
impl ScriptedHost {
|
||||
fn failing(group: &str, failures: Vec<AttemptError>) -> Self {
|
||||
Self {
|
||||
script: Mutex::new(vec![(group.to_owned(), failures.into())]),
|
||||
..Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Host for ScriptedHost {
|
||||
async fn invoke(&self, attempt: &Attempt) -> Result<Output, AttemptError> {
|
||||
let next = self
|
||||
.script
|
||||
.lock()
|
||||
.iter_mut()
|
||||
.find(|(group, _)| *group == attempt.model_group)
|
||||
.and_then(|(_, failures)| failures.pop_front());
|
||||
match next {
|
||||
Some(failure) => Err(failure),
|
||||
None => Ok(Output {
|
||||
value: Payload(Arc::new(attempt.deployment.id.clone())),
|
||||
total_tokens: 10,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn select(
|
||||
&self,
|
||||
_stage: &SelectStage,
|
||||
_ctx: &RoutingContext<'_>,
|
||||
candidates: Vec<Arc<Deployment>>,
|
||||
) -> Result<Vec<Arc<Deployment>>, Error> {
|
||||
Ok(candidates)
|
||||
}
|
||||
|
||||
async fn allow_fallback(&self, _request: &RouterRequest, _target: &str) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
async fn sleep(&self, _duration: Duration) {}
|
||||
|
||||
fn observe(&self, event: &RouterEvent) {
|
||||
self.events.lock().push(event.clone());
|
||||
}
|
||||
|
||||
fn cache(&self) -> Arc<dyn RouterCache> {
|
||||
Arc::new(self.cache.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[fixture]
|
||||
fn config() -> RouterConfig {
|
||||
serde_json::from_value(json!({
|
||||
"model_list": [
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/a"}, "model_info": {"id": "a1"}},
|
||||
{"model_name": "primary", "litellm_params": {"model": "openai/b"}, "model_info": {"id": "a2"}},
|
||||
{"model_name": "backup", "litellm_params": {"model": "anthropic/c"}, "model_info": {"id": "b1"}},
|
||||
]
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn router(config: RouterConfig, dynamic: serde_json::Value, host: Arc<ScriptedHost>) -> Router {
|
||||
let dynamic: RouterDynamicConfig = serde_json::from_value(dynamic).unwrap();
|
||||
Router::builder(config, dynamic, host)
|
||||
.randomness(Arc::new(SeededRandomness::new(7)))
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn request() -> RouterRequest {
|
||||
RouterRequest {
|
||||
model: "primary".to_owned(),
|
||||
..RouterRequest::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_rate_limit_retries_in_the_same_group(config: RouterConfig) {
|
||||
let host = Arc::new(ScriptedHost::failing(
|
||||
"primary",
|
||||
vec![AttemptError::new(FailureKind::RateLimit, Some(429))],
|
||||
));
|
||||
let router = router(config, json!({"num_retries": 2}), host.clone());
|
||||
let response = router.call(Op::Completion, request()).await.unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
response.model_group.as_str(),
|
||||
response.attempted_retries,
|
||||
response.attempted_fallbacks
|
||||
),
|
||||
("primary", 1, 0)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_bad_request_falls_back_without_retrying(config: RouterConfig) {
|
||||
let host = Arc::new(ScriptedHost::failing(
|
||||
"primary",
|
||||
vec![AttemptError::new(FailureKind::BadRequest, Some(400))],
|
||||
));
|
||||
let router = router(
|
||||
config,
|
||||
json!({"num_retries": 3, "fallbacks": [{"primary": ["backup"]}]}),
|
||||
host.clone(),
|
||||
);
|
||||
let response = router.call(Op::Completion, request()).await.unwrap();
|
||||
let retried = host
|
||||
.events
|
||||
.lock()
|
||||
.iter()
|
||||
.any(|event| matches!(event, RouterEvent::Retry { .. }));
|
||||
assert_eq!(
|
||||
(
|
||||
response.deployment_id.as_str(),
|
||||
response.attempted_fallbacks,
|
||||
retried
|
||||
),
|
||||
("b1", 1, false)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_rate_limit_cools_the_deployment_down(config: RouterConfig) {
|
||||
let host = Arc::new(ScriptedHost::failing(
|
||||
"primary",
|
||||
vec![AttemptError::new(FailureKind::RateLimit, Some(429))],
|
||||
));
|
||||
let router = router(config, json!({}), host.clone());
|
||||
let error = router.call(Op::Completion, request()).await.unwrap_err();
|
||||
let cooled = router
|
||||
.health()
|
||||
.active_cooldowns(&["a1".to_owned(), "a2".to_owned()])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(error.attempt().map(|failure| failure.kind), cooled.len()),
|
||||
(Some(FailureKind::RateLimit), 1)
|
||||
);
|
||||
}
|
||||
|
|
@ -7,8 +7,9 @@ the router is built, from its ``RouterConfig``.
|
|||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.router_config import RouterConfig
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -51,6 +52,28 @@ class RouterCore(Protocol):
|
|||
|
||||
|
||||
def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
|
||||
"""
|
||||
The Rust core when the route catalog's router rule allows it and the Rust core serves this
|
||||
config, else the Python core. Decided once, for the router's life.
|
||||
"""
|
||||
from litellm.router_python_core import PythonCore
|
||||
from litellm.rust_bridge.catalog import Route, RouteContext, decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
||||
return (core_class or PythonCore)(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature
|
||||
python_class: Final = core_class or PythonCore
|
||||
chosen: Final = decision(RouteContext(Route.ROUTER))
|
||||
if chosen is not Decision.PYTHON:
|
||||
from litellm.router_rust_core import RustCore, RustCoreDeclined
|
||||
from litellm.rust_bridge._native import RustBridgeDeclined
|
||||
|
||||
try:
|
||||
return RustCore.build(config, host, python_class)
|
||||
except (RustBridgeDeclined, RustCoreDeclined) as declined:
|
||||
# A config the Rust core cannot take keeps the router on Python; an error building the
|
||||
# Python core itself is the router's own error and is raised once, as it is.
|
||||
if chosen is Decision.RUST_REQUIRED:
|
||||
raise
|
||||
verbose_router_logger.info("Router served by the Python core: %s", declined)
|
||||
if isinstance(declined, RustCoreDeclined):
|
||||
return declined.python
|
||||
return python_class(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature
|
||||
|
|
|
|||
813
litellm/router_rust_core.py
Normal file
813
litellm/router_rust_core.py
Normal file
|
|
@ -0,0 +1,813 @@
|
|||
"""
|
||||
``RustCore``: the router core served by litellm-rust/crates/router-core through ``_native``.
|
||||
|
||||
The Rust core owns the registry, selection, retries, fallbacks and cooldowns for the operations in
|
||||
``RUST_SERVED_OPS``. It never awaits Python: ``RouterCall.next`` returns the step the call loop is
|
||||
blocked on (run an attempt on a deployment, sleep, check a fallback target, run a plugin) and this
|
||||
driver answers it. Each attempt runs through the Python core with that one deployment, so request
|
||||
preparation, inference, logging and cost tracking are unchanged.
|
||||
|
||||
Everything else (the views, runtime writes, every other endpoint) is served by the ``PythonCore``
|
||||
the Rust core wraps, built from the same config. A config the Rust core cannot serve fails at
|
||||
construction and ``make_core`` builds a ``PythonCore`` instead.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_router_logger
|
||||
from litellm.integrations.custom_guardrail import is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.add_retry_fallback_headers import (
|
||||
add_fallback_headers_to_response,
|
||||
add_retry_headers_to_response,
|
||||
)
|
||||
from litellm.litellm_core_utils.routing_common_utils import (
|
||||
format_content_policy_fallback_hint,
|
||||
format_fallback_outcome_message,
|
||||
format_no_fallback_group_message,
|
||||
truncate_fallback_error_detail,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_string
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure
|
||||
from litellm.router_config import RouterConfig
|
||||
from litellm.router_python_core import PythonCore
|
||||
from litellm.router_python_host import PythonHost
|
||||
from litellm.router_utils.fallback_event_handlers import (
|
||||
AttemptedFallbackTargets,
|
||||
_provider_prefixed_model_group,
|
||||
fallback_lookup_groups,
|
||||
get_fallback_model_group_for_lookup_groups,
|
||||
is_bad_request_without_fallback,
|
||||
)
|
||||
from litellm.types.router import FallbackSkipReason
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.rust_bridge._native import RouterCall, RouterCoreHandle
|
||||
|
||||
RustServedOp: TypeAlias = Literal["acompletion", "aanthropic_messages", "aembedding", "aresponses"]
|
||||
RUST_SERVED_OPS: Final[frozenset[str]] = frozenset({"acompletion", "aanthropic_messages", "aembedding", "aresponses"})
|
||||
|
||||
_FAILURE_KINDS: Final = (
|
||||
"RateLimitError",
|
||||
"ContextWindowExceededError",
|
||||
"ContentPolicyViolationError",
|
||||
"AuthenticationError",
|
||||
"PermissionDeniedError",
|
||||
"NotFoundError",
|
||||
"BadRequestError",
|
||||
"Timeout",
|
||||
"APIConnectionError",
|
||||
"InternalServerError",
|
||||
"ServiceUnavailableError",
|
||||
"BadGatewayError",
|
||||
"GuardrailIntervention",
|
||||
"CallerCredentialError",
|
||||
)
|
||||
_DECLINED: Final = frozenset({"MalformedRequestFieldError", "UnsupportedEndpointError"})
|
||||
_STEP: Final = TypeAdapter(Mapping[str, object])
|
||||
_REQUEST_FIELDS: Final = ("specific_deployment", "num_retries", "disable_fallbacks")
|
||||
# Request keys router-core reads under another name.
|
||||
_RENAMED_REQUEST_FIELDS: Final = MappingProxyType({"_router_weights": "router_weights"})
|
||||
_PROVIDERS: Final = tuple(provider.value for provider in LlmProviders)
|
||||
_PYTHON_ONLY_REQUEST_KEYS: Final = frozenset(
|
||||
{
|
||||
"fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"default_fallbacks",
|
||||
"max_fallbacks",
|
||||
"model_group_retry_policy",
|
||||
"retry_policy",
|
||||
}
|
||||
)
|
||||
_STATIC_FIELDS: Final = (
|
||||
"routing_strategy",
|
||||
"cooldown_time",
|
||||
"disable_cooldowns",
|
||||
"allowed_fails_policy",
|
||||
"retry_after",
|
||||
"timeout",
|
||||
)
|
||||
_HANDLED_FIELDS: Final = frozenset(
|
||||
{
|
||||
*_STATIC_FIELDS,
|
||||
"model_list",
|
||||
"model_group_alias",
|
||||
"fallbacks",
|
||||
"default_fallbacks",
|
||||
"context_window_fallbacks",
|
||||
"content_policy_fallbacks",
|
||||
"num_retries",
|
||||
"max_fallbacks",
|
||||
"allowed_fails",
|
||||
"set_verbose",
|
||||
"debug_level",
|
||||
}
|
||||
)
|
||||
_PYTHON_DEFAULTS: Final = MappingProxyType(
|
||||
{
|
||||
name: parameter.default
|
||||
for name, parameter in inspect.signature(PythonCore.__init__).parameters.items()
|
||||
if parameter.default is not inspect.Parameter.empty
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def failure_kind(error: BaseException) -> str:
|
||||
names: Final = tuple(cls.__name__ for cls in type(error).__mro__)
|
||||
if not _DECLINED.isdisjoint(names):
|
||||
return "Declined"
|
||||
if is_guardrail_intervention(error): # pyright: ignore[reportArgumentType] # every attempt error is an Exception
|
||||
return "GuardrailIntervention"
|
||||
return next((name for name in names if name in _FAILURE_KINDS), "APIError")
|
||||
|
||||
|
||||
def _retry_after_seconds(error: BaseException) -> float | None:
|
||||
response: Final = getattr(error, "response", None)
|
||||
headers: Final = getattr(error, "litellm_response_headers", None) or getattr(response, "headers", None)
|
||||
if headers is None or not hasattr(headers, "get"):
|
||||
return None
|
||||
milliseconds: Final = headers.get("retry-after-ms")
|
||||
seconds: Final = headers.get("retry-after")
|
||||
try:
|
||||
value: Final = float(milliseconds) / 1000 if milliseconds is not None else float(seconds)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return value if math.isfinite(value) and value >= 0 else None
|
||||
|
||||
|
||||
def attempt_failure(error: BaseException) -> Mapping[str, object]:
|
||||
status: Final = getattr(error, "status_code", None)
|
||||
num_retries: Final = getattr(error, "num_retries", None)
|
||||
return {
|
||||
"ok": False,
|
||||
"kind": failure_kind(error),
|
||||
"status": status if isinstance(status, int) and 100 <= status <= 599 else None,
|
||||
"retry_after": _retry_after_seconds(error),
|
||||
"num_retries": num_retries if isinstance(num_retries, int) and num_retries >= 0 else None,
|
||||
"detail": str(error)[:500],
|
||||
}
|
||||
|
||||
|
||||
def total_tokens(response: object) -> int:
|
||||
usage: Final = getattr(response, "usage", None)
|
||||
tokens: Final = getattr(usage, "total_tokens", None)
|
||||
return tokens if isinstance(tokens, int) else 0
|
||||
|
||||
|
||||
def _json_default(value: object) -> object:
|
||||
return str(value)
|
||||
|
||||
|
||||
def _plain(value: object) -> object:
|
||||
"""``value`` as plain JSON values, the shape the Rust core reads."""
|
||||
return json.loads(json.dumps(value, default=_json_default))
|
||||
|
||||
|
||||
# Where an applied static setting lands on the Python core: the value the Rust core is rebuilt with.
|
||||
_APPLIED_STATIC: Final = MappingProxyType(
|
||||
{
|
||||
"routing_strategy": lambda python: python.settings().routing_strategy,
|
||||
"cooldown_time": lambda python: python.settings().cooldown_time,
|
||||
"disable_cooldowns": lambda python: python.settings().disable_cooldowns,
|
||||
"allowed_fails_policy": lambda python: python.settings().allowed_fails_policy,
|
||||
"retry_after": lambda python: python.retry_after,
|
||||
"timeout": lambda python: python.timeout,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _static_value(value: object) -> object:
|
||||
dump: Final = getattr(value, "model_dump", None)
|
||||
return _plain(dump(exclude_none=True) if callable(dump) else value)
|
||||
|
||||
|
||||
def static_settings(config: RouterConfig, python: PythonCore) -> Mapping[str, object]:
|
||||
"""The static settings: the constructor arguments, overlaid with every setting a changeset
|
||||
(``apply``) has changed since, as the Python core resolved it."""
|
||||
applied: Final = python._core_applied
|
||||
values: Final = {name: config.arguments[name] for name in _STATIC_FIELDS if name in config.arguments}
|
||||
values.update({name: read(python) for name, read in _APPLIED_STATIC.items() if name in applied})
|
||||
return {name: _static_value(value) for name, value in values.items() if value is not None}
|
||||
|
||||
|
||||
def static_document(config: RouterConfig, python: PythonCore) -> Mapping[str, object]:
|
||||
"""The Python core's own normalized deployments (with their ids) and the static settings."""
|
||||
return {"model_list": _plain(python._core_model_list), **static_settings(config, python)}
|
||||
|
||||
|
||||
def dynamic_document(python: PythonCore) -> Mapping[str, object]:
|
||||
"""The values the Python core resolved from its arguments and the process settings."""
|
||||
settings: Final = python.settings()
|
||||
resolved: Final = {
|
||||
"version": python._registry_version,
|
||||
"model_group_alias": python._core_model_group_alias,
|
||||
"fallbacks": settings.fallbacks,
|
||||
"context_window_fallbacks": settings.context_window_fallbacks,
|
||||
"content_policy_fallbacks": settings.content_policy_fallbacks,
|
||||
"num_retries": python.num_retries,
|
||||
"max_fallbacks": python.max_fallbacks,
|
||||
"allowed_fails": python._core_allowed_fails
|
||||
if "allowed_fails" in python._core_applied
|
||||
else python.explicit_allowed_fails,
|
||||
"fallback_keys": fallback_keys(python, settings),
|
||||
}
|
||||
return {name: _plain(value) for name, value in resolved.items() if value is not None}
|
||||
|
||||
|
||||
def fallback_keys(python: PythonCore, settings: object) -> Mapping[str, object]:
|
||||
"""The spellings a fallback key matches besides the group name (`get_fallback_model_group`)."""
|
||||
lists: Final = (
|
||||
getattr(settings, "fallbacks", None),
|
||||
getattr(settings, "context_window_fallbacks", None),
|
||||
getattr(settings, "content_policy_fallbacks", None),
|
||||
)
|
||||
keys: Final = tuple( # comprehension-ok: the first group of each fallback entry, across the three lists
|
||||
next(iter(entry))
|
||||
for entries in lists
|
||||
if isinstance(entries, list)
|
||||
for entry in entries
|
||||
if isinstance(entry, dict) and entry
|
||||
)
|
||||
names: Final = {*(python._core_model_names or ()), *(python._core_model_group_alias or {})}
|
||||
prefixed: Final = {
|
||||
name: spelled for name in names if (spelled := _provider_prefixed_model_group(name, keys)) is not None
|
||||
}
|
||||
return {"providers": _PROVIDERS, "provider_prefixed": prefixed}
|
||||
|
||||
|
||||
def unsupported_arguments(config: RouterConfig) -> frozenset[str]:
|
||||
"""Arguments the Rust core does not serve; any of them keeps the router on Python."""
|
||||
changed: Final = frozenset(
|
||||
name
|
||||
for name, value in config.arguments.items()
|
||||
if name not in _HANDLED_FIELDS and value != _PYTHON_DEFAULTS.get(name, None)
|
||||
)
|
||||
strategy: Final = config.arguments.get("routing_strategy", "simple-shuffle")
|
||||
model_list: Final = config.arguments.get("model_list")
|
||||
# `simple_shuffle` raises on a non-numeric weight when it picks; the Python core keeps that.
|
||||
odd_weight: Final = isinstance(model_list, list) and any(
|
||||
isinstance(params := (deployment.get("litellm_params") if isinstance(deployment, Mapping) else None), Mapping)
|
||||
and params.get("weight") is not None
|
||||
and (isinstance(params.get("weight"), bool) or not isinstance(params.get("weight"), int | float))
|
||||
for deployment in model_list
|
||||
)
|
||||
return (
|
||||
changed
|
||||
| ({"routing_strategy"} if strategy != "simple-shuffle" else frozenset())
|
||||
| ({"model_list"} if odd_weight else frozenset())
|
||||
)
|
||||
|
||||
|
||||
def runs_on_python(request: Mapping[str, object]) -> bool:
|
||||
"""Per-request routing overrides the Rust core does not read yet."""
|
||||
return any(key in _PYTHON_ONLY_REQUEST_KEYS or key.startswith("mock_testing_") for key in request)
|
||||
|
||||
|
||||
def request_team_id(request: Mapping[str, object]) -> str | None:
|
||||
metadata: Final = request.get("metadata") or request.get("litellm_metadata")
|
||||
team_id: Final = metadata.get("user_api_key_team_id") if isinstance(metadata, Mapping) else None
|
||||
return team_id if isinstance(team_id, str) else None
|
||||
|
||||
|
||||
def _bound_request(bound: inspect.BoundArguments) -> Mapping[str, object]:
|
||||
"""The keyword payload of a call bound to the Python core's endpoint signature."""
|
||||
named: Final = {name: value for name, value in bound.arguments.items() if name not in ("self", "kwargs")}
|
||||
extra: Final = bound.arguments.get("kwargs")
|
||||
return {**named, **(extra if isinstance(extra, Mapping) else {})}
|
||||
|
||||
|
||||
_ACOMPLETION: Final = inspect.signature(PythonCore.acompletion)
|
||||
_AEMBEDDING: Final = inspect.signature(PythonCore.aembedding)
|
||||
_OWN_ATTRIBUTES: Final = frozenset({"_handle", "_config", "_python", "_host", "_built_for"})
|
||||
|
||||
|
||||
class RustCoreDeclined(Exception):
|
||||
"""The Rust core cannot serve a router whose Python core is already built: the router keeps that one."""
|
||||
|
||||
def __init__(self, reason: str, python: PythonCore) -> None:
|
||||
super().__init__(reason)
|
||||
self.python = python
|
||||
|
||||
|
||||
class RustCore:
|
||||
"""A ``RouterCore`` whose routing for ``RUST_SERVED_OPS`` runs in Rust, over a ``PythonCore``."""
|
||||
|
||||
def __init__(self, handle: "RouterCoreHandle", config: RouterConfig, python: PythonCore, host: PythonHost) -> None:
|
||||
self._handle: RouterCoreHandle | None = handle
|
||||
self._config: Final = config
|
||||
self._python: Final = python
|
||||
self._host: Final = host
|
||||
self._built_for: tuple[int, int] = self._state()
|
||||
|
||||
@classmethod
|
||||
def build(cls, config: RouterConfig, host: PythonHost, python_class: type[PythonCore] = PythonCore) -> "RustCore":
|
||||
from litellm.rust_bridge._native import RouterCoreHandle, RustBridgeDeclined
|
||||
|
||||
extra: Final = unsupported_arguments(config)
|
||||
if extra:
|
||||
raise RustBridgeDeclined(f"router arguments not served by the Rust core: {sorted(extra)}")
|
||||
python: Final = python_class(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the arguments were bound against PythonCore's own signature
|
||||
try:
|
||||
handle: Final = RouterCoreHandle(static_document(config, python), dynamic_document(python))
|
||||
except Exception as declined:
|
||||
# The Python core is built already (its callbacks registered): keep it, don't build another.
|
||||
raise RustCoreDeclined(str(declined), python) from declined
|
||||
return cls(handle, config, python, host)
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
return getattr(self._python, name)
|
||||
|
||||
def __setattr__(self, name: str, value: object) -> None:
|
||||
if name in _OWN_ATTRIBUTES:
|
||||
object.__setattr__(self, name, value)
|
||||
return
|
||||
setattr(self._python, name, value)
|
||||
|
||||
def __delattr__(self, name: str) -> None:
|
||||
if name in _OWN_ATTRIBUTES:
|
||||
object.__delattr__(self, name)
|
||||
return
|
||||
delattr(self._python, name)
|
||||
|
||||
def _state(self) -> tuple[int, int]:
|
||||
"""What the handle was built from: the deployments, and the settings changesets applied."""
|
||||
return (self._python._registry_version, self._python._settings_version)
|
||||
|
||||
def _current_handle(self) -> "RouterCoreHandle | None":
|
||||
"""The Rust handle, rebuilt after a deployment change or an applied changeset; ``None`` while
|
||||
the Rust core cannot serve the router."""
|
||||
if "_core_async_get_available_deployment" in vars(self._python):
|
||||
# `register_plugin` installed a custom strategy, which only the Python core runs.
|
||||
return None
|
||||
if self._state() == self._built_for:
|
||||
return self._handle
|
||||
from litellm.rust_bridge._native import RouterCoreHandle
|
||||
|
||||
if self._handle is not None:
|
||||
self._handle.retire()
|
||||
strategy: Final = self._python.settings().routing_strategy
|
||||
try:
|
||||
if strategy not in (None, "simple-shuffle"):
|
||||
raise ValueError(f"routing_strategy {strategy!r} is not served by the Rust core")
|
||||
self._handle = RouterCoreHandle(static_document(self._config, self._python), dynamic_document(self._python))
|
||||
except Exception as declined: # noqa: BLE001 # a change the Rust core cannot take keeps the router on Python until the next one
|
||||
verbose_router_logger.info("Router served by the Python core after a change: %s", declined)
|
||||
self._handle = None
|
||||
self._built_for = self._state()
|
||||
return self._handle
|
||||
|
||||
async def call(self, op: str, request: Mapping[str, object]) -> object:
|
||||
if op not in RUST_SERVED_OPS:
|
||||
return await self._python.call(op, request)
|
||||
return await self._drive(op, request)
|
||||
|
||||
async def acompletion(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload
|
||||
try:
|
||||
bound: Final = _ACOMPLETION.bind(self._python, *args, **kwargs)
|
||||
except TypeError:
|
||||
# The Python core raises the caller's mistake in its own words.
|
||||
return await self._python.acompletion(*args, **kwargs) # pyright: ignore[reportCallIssue] # the call is expected to raise
|
||||
return await self._drive("acompletion", _bound_request(bound))
|
||||
|
||||
async def aanthropic_messages(self, **kwargs: object) -> object: # kwargs-ok: the endpoint's own keyword payload
|
||||
return await self._drive("aanthropic_messages", kwargs)
|
||||
|
||||
async def aembedding(self, *args: object, **kwargs: object) -> object: # kwargs-ok: endpoint payload
|
||||
try:
|
||||
bound: Final = _AEMBEDDING.bind(self._python, *args, **kwargs)
|
||||
except TypeError:
|
||||
return await self._python.aembedding(*args, **kwargs) # pyright: ignore[reportCallIssue] # the call is expected to raise
|
||||
return await self._drive("aembedding", _bound_request(bound))
|
||||
|
||||
async def aresponses(self, **kwargs: object) -> object: # kwargs-ok: the endpoint's own keyword payload
|
||||
return await self._drive("aresponses", kwargs)
|
||||
|
||||
def retire(self) -> None:
|
||||
if self._handle is not None:
|
||||
self._handle.retire()
|
||||
self._python.retire()
|
||||
|
||||
async def _drive(self, op: str, request: Mapping[str, object]) -> object:
|
||||
handle: Final = self._current_handle()
|
||||
if handle is None or runs_on_python(request) or not isinstance(request.get("model"), str):
|
||||
return await self._python.call(op, request)
|
||||
if not hasattr(request.get(_METADATA_FIELDS.get(op, "metadata"), {}), "update"):
|
||||
# `_update_kwargs_before_fallbacks` raises on metadata it cannot update, before any attempt.
|
||||
return await self._python.call(op, request)
|
||||
fields: Final = {
|
||||
**{key: request[key] for key in _REQUEST_FIELDS if request.get(key) is not None},
|
||||
**{name: request[key] for key, name in _RENAMED_REQUEST_FIELDS.items() if request.get(key) is not None},
|
||||
}
|
||||
team_id: Final = request_team_id(request)
|
||||
try:
|
||||
call: Final = handle.start(
|
||||
op, _plain({"model": request["model"], **fields, **({"team_id": team_id} if team_id else {})})
|
||||
)
|
||||
except ValueError:
|
||||
return await self._python.call(op, request)
|
||||
return await _Driver(call, self._python, op, request).run()
|
||||
|
||||
|
||||
# The request field `_update_kwargs_before_fallbacks` writes the model group into, per op.
|
||||
_METADATA_FIELDS: Final = MappingProxyType(
|
||||
{"aanthropic_messages": "litellm_metadata", "aresponses": "litellm_metadata"}
|
||||
)
|
||||
|
||||
|
||||
def _ttl(step: Mapping[str, object]) -> Mapping[str, object]:
|
||||
ttl: Final = step.get("ttl")
|
||||
return {"ttl": ttl} if isinstance(ttl, int | float) else {}
|
||||
|
||||
|
||||
def _store_for(python: PythonCore, key: str) -> "DualCache":
|
||||
"""Cooldowns live in the router's cooldown store; every other routing key in its main DualCache."""
|
||||
health: Final = python.health()
|
||||
return health.cooldowns.cooldown_store if key.endswith(":cooldown") else health.store
|
||||
|
||||
|
||||
async def answer_cache_step(python: PythonCore, step: Mapping[str, object]) -> object:
|
||||
"""Run one cache op the Rust core asked for against the router's own DualCaches (memory plus Redis)."""
|
||||
key: Final = str(step.get("key"))
|
||||
match step.get("op"):
|
||||
case "get":
|
||||
return _plain(await _store_for(python, key).async_get_cache(key))
|
||||
case "batch_get":
|
||||
keys: Final = [str(k) for k in step.get("keys") or ()]
|
||||
values: Final = [await _store_for(python, k).async_get_cache(k) for k in keys]
|
||||
return _plain(values)
|
||||
case "set":
|
||||
await _store_for(python, key).async_set_cache(key, step.get("value"), **_ttl(step))
|
||||
return None
|
||||
case "increment":
|
||||
amount: Final = step.get("value")
|
||||
by: Final = amount if isinstance(amount, int | float) and not isinstance(amount, bool) else 1
|
||||
total: Final = await _store_for(python, key).async_increment_cache(key, by, **_ttl(step))
|
||||
return total if isinstance(total, int | float) else by
|
||||
case "delete":
|
||||
await _store_for(python, key).async_delete_cache(key)
|
||||
return None
|
||||
case other:
|
||||
raise RuntimeError(f"router-core asked for an unknown cache op {other!r}")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Hop:
|
||||
"""One target of a fallback walk and what its level ended on."""
|
||||
|
||||
group: str
|
||||
pending: Mapping[str, object] | None
|
||||
error: BaseException | None = None
|
||||
failed: Mapping[str, object] | None = None
|
||||
children: "list[_Hop]" = field(default_factory=list) # mutable-ok: the fallbacks tried from this target
|
||||
|
||||
|
||||
class _Driver:
|
||||
"""Answers the steps one Rust call loop blocks on until it is done.
|
||||
|
||||
Each attempt runs on the Python core with retries and fallbacks off, so the driver puts back
|
||||
what Python's own retry and fallback layers add to the outcome: the retry stamp on an error
|
||||
whose retry loop ran out, the retry and fallback headers on a response, and the explanation
|
||||
the fallback layer appends to the error it finally raises.
|
||||
"""
|
||||
|
||||
def __init__(self, call: "RouterCall", python: PythonCore, op: str, request: Mapping[str, object]) -> None:
|
||||
self._call: Final = call
|
||||
self._python: Final = python
|
||||
self._op: Final = op
|
||||
self._request: Final = request
|
||||
self._last_response: object = None
|
||||
self._last_error: BaseException | None = None
|
||||
# The last failure at each fallback level: a level raises its own last error.
|
||||
self._level_errors: Final[dict[int, BaseException]] = {} # mutable-ok: filled as the call loop runs
|
||||
self._attempted: bool = False
|
||||
self._walk: Final = AttemptedFallbackTargets()
|
||||
# Fallback targets in the order Rust walked them, and the groups an attempt was made in.
|
||||
self._hop_targets: Final[list[str]] = [] # mutable-ok: filled as the call loop runs
|
||||
self._attempted_groups: Final[set[str]] = set() # mutable-ok: filled as the call loop runs
|
||||
# The walk in order: ("hop", (level, group, order)), ("attempt", level), ("error", (level,
|
||||
# error, request)), ("sleep", None) before each retry, and ("exhausted", group) when a retry
|
||||
# loop ran out.
|
||||
self._timeline: Final[list[tuple[str, object]]] = [] # mutable-ok: filled as the call loop runs
|
||||
|
||||
async def run(self) -> object:
|
||||
outcome: object = None # rebind-ok: each step's outcome answers it
|
||||
while True:
|
||||
step: Final = _STEP.validate_python(self._call.next(outcome))
|
||||
self._take_events(step.get("events"))
|
||||
match step["kind"]:
|
||||
case "attempt":
|
||||
outcome = await self._attempt(step)
|
||||
case "sleep":
|
||||
self._timeline.append(("sleep", None))
|
||||
await self._python._host.sleep(float(str(step["seconds"])))
|
||||
outcome = None
|
||||
case "allow_fallback":
|
||||
outcome = await self._allow_fallback(str(step["target"]))
|
||||
case "cache":
|
||||
outcome = await answer_cache_step(self._python, step)
|
||||
case "select":
|
||||
outcome = step["candidates"]
|
||||
case "done":
|
||||
return self._with_headers(step)
|
||||
case "failed":
|
||||
primary: Final = self._level_errors.get(0)
|
||||
if primary is not None:
|
||||
if self._walks_order_tiers(primary, self._request.get("model")):
|
||||
# Python's order-tier walk raises what its last fallback raised, as is.
|
||||
raise await self._walk_error(self._walk_tree(), primary)
|
||||
await self._fill_unattempted_level()
|
||||
self._explain(primary)
|
||||
raise primary
|
||||
if not self._attempted:
|
||||
# No deployment was tried (unknown model, nothing healthy): Python raises its own error.
|
||||
return await self._python.call(self._op, self._request)
|
||||
raise RuntimeError(f"router-core ({step.get('error_kind')}): {step.get('error')}")
|
||||
case other:
|
||||
raise RuntimeError(f"router-core returned an unknown step {other!r}")
|
||||
|
||||
def _take_events(self, events: object) -> None:
|
||||
for event in events if isinstance(events, list) else ():
|
||||
if isinstance(event, Mapping) and event.get("kind") == "fallback":
|
||||
self._timeline.append(("hop", (event.get("level"), event.get("to"), event.get("order"))))
|
||||
if not isinstance(event, Mapping) or event.get("kind") != "retries_exhausted":
|
||||
continue
|
||||
self._timeline.append(("exhausted", event.get("model_group")))
|
||||
error: BaseException | None = self._last_error
|
||||
# `async_function_with_retries` stamps the error its exhausted retry loop raises.
|
||||
if error is not None and type(error) in self._python._host.dynamic().exception_types:
|
||||
error.max_retries = event.get("max") # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes
|
||||
error.num_retries = event.get("attempted") # pyright: ignore[reportAttributeAccessIssue] # Python's router stamps the same attributes
|
||||
|
||||
def _walks_order_tiers(self, error: BaseException, group: object) -> bool:
|
||||
"""`async_function_with_fallbacks_common_utils` hands a group with more than one order level
|
||||
to `run_async_fallback` outside its try, so the walk's own error is raised with no suffix."""
|
||||
import litellm.exceptions as litellm_exceptions
|
||||
|
||||
python: Final = self._python
|
||||
request: Final = self._request
|
||||
if (
|
||||
request.get("disable_fallbacks") is True
|
||||
or not isinstance(group, str)
|
||||
or is_guardrail_intervention(error) # pyright: ignore[reportArgumentType] # every attempt error is an Exception
|
||||
or is_bad_request_without_fallback(error, python.fallback_on_unsupported_params) # pyright: ignore[reportArgumentType] # every attempt error is an Exception
|
||||
or isinstance(
|
||||
error,
|
||||
litellm_exceptions.ContextWindowExceededError | litellm_exceptions.ContentPolicyViolationError,
|
||||
)
|
||||
):
|
||||
return False
|
||||
metadata: Final = request.get("metadata")
|
||||
team: Final = metadata.get("user_api_key_team_id") if isinstance(metadata, Mapping) else None
|
||||
deployments: Final = python._core_get_model_list(model_name=group, team_id=team) or ()
|
||||
orders: Final = {
|
||||
order
|
||||
for deployment in deployments
|
||||
if (order := (deployment.get("litellm_params") or {}).get("order")) is not None
|
||||
}
|
||||
return len(orders) > 1
|
||||
|
||||
def _walk_tree(self) -> _Hop:
|
||||
"""The fallback walk as Rust ran it: each hop under the level it was tried from."""
|
||||
root: Final = _Hop(str(self._request.get("model")), None)
|
||||
current: Final[dict[int, _Hop]] = {0: root} # mutable-ok: the newest hop at each level
|
||||
recent: _Hop | None = None # rebind-ok: the hop whose attempt failed last
|
||||
for kind, value in self._timeline:
|
||||
if kind == "hop" and isinstance(value, tuple):
|
||||
level, group, order = value
|
||||
hop = _Hop(str(group), {**self._request, "model": str(group)})
|
||||
if isinstance(order, int):
|
||||
hop.pending = {**self._request, "model": str(group), "_target_order": order}
|
||||
parent = current.get(level - 1 if isinstance(level, int) else -1)
|
||||
if parent is not None and isinstance(level, int):
|
||||
parent.children.append(hop)
|
||||
current[level] = hop
|
||||
recent = None
|
||||
elif kind == "attempt" and isinstance(value, int) and value in current:
|
||||
current[value].pending = None
|
||||
elif kind == "error" and isinstance(value, tuple) and value[0] in current:
|
||||
recent = current[value[0]]
|
||||
recent.error, recent.failed = value[1], value[2]
|
||||
elif kind == "sleep" and recent is not None:
|
||||
recent.pending = recent.failed
|
||||
elif kind == "exhausted" and recent is not None and recent.pending is recent.failed:
|
||||
recent.pending = None
|
||||
return root
|
||||
|
||||
async def _walk_error(self, hop: _Hop, primary: BaseException) -> BaseException:
|
||||
"""The error a hop's level raises. Python raises its own error where Rust made no attempt:
|
||||
for a target with no deployment, and for a retry that found no deployment left to try. A
|
||||
group with more than one order level raises its walk's last error instead
|
||||
(`run_async_fallback`), whatever that walk tried."""
|
||||
own: Final = await self._own_error(hop, primary)
|
||||
if not hop.children or not self._walks_order_tiers(own, hop.group):
|
||||
return own
|
||||
last: BaseException = own # rebind-ok: each fallback tried replaces it
|
||||
for child in hop.children:
|
||||
last = await self._walk_error(child, primary)
|
||||
return last
|
||||
|
||||
async def _own_error(self, hop: _Hop, primary: BaseException) -> BaseException:
|
||||
"""The error the hop's own target raised: Python's, replayed, when Rust made no attempt."""
|
||||
if hop.pending is not None:
|
||||
try:
|
||||
await self._python.call(self._op, {**hop.pending, "num_retries": 0, "disable_fallbacks": True})
|
||||
except Exception as error: # noqa: BLE001 # the error Python's walk would end on
|
||||
return error
|
||||
return hop.error or primary
|
||||
|
||||
async def _fill_unattempted_level(self) -> None:
|
||||
"""A first fallback that resolved no deployment made no attempt, so Rust holds no error for
|
||||
it: ask Python, which raises its own no-deployment error without calling a provider."""
|
||||
if 1 in self._level_errors:
|
||||
return
|
||||
target: Final = next((t for t in self._hop_targets if t not in self._attempted_groups), None)
|
||||
if target is None:
|
||||
return
|
||||
try:
|
||||
await self._python.call(
|
||||
self._op, {**self._request, "model": target, "num_retries": 0, "disable_fallbacks": True}
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # the error is only quoted in the primary error's message
|
||||
self._level_errors[1] = error
|
||||
|
||||
def _with_headers(self, step: Mapping[str, object]) -> object:
|
||||
response: object = self._last_response # rebind-ok: the response gains its headers in turn
|
||||
retries: Final = step.get("attempted_retries")
|
||||
if isinstance(retries, int) and retries > 0:
|
||||
max_retries: Final = step.get("max_retries")
|
||||
response = add_retry_headers_to_response( # rebind-ok: the response gains its headers in turn
|
||||
response=response,
|
||||
attempted_retries=retries,
|
||||
max_retries=max_retries if isinstance(max_retries, int) else None,
|
||||
)
|
||||
fallbacks: Final = step.get("attempted_fallbacks")
|
||||
return add_fallback_headers_to_response(
|
||||
response=response, attempted_fallbacks=fallbacks if isinstance(fallbacks, int) else 0
|
||||
)
|
||||
|
||||
def _explain(self, error: BaseException) -> None:
|
||||
"""Append what `async_function_with_fallbacks_common_utils` appends to the error it raises."""
|
||||
python: Final = self._python
|
||||
request: Final = self._request
|
||||
if (
|
||||
request.get("disable_fallbacks") is True
|
||||
or is_guardrail_intervention(error) # pyright: ignore[reportArgumentType] # every attempt error is an Exception
|
||||
or is_bad_request_without_fallback(error, python.fallback_on_unsupported_params) # pyright: ignore[reportArgumentType] # every attempt error is an Exception
|
||||
or not hasattr(error, "message")
|
||||
or not python._host.dynamic().expose_router_debug_in_errors
|
||||
):
|
||||
return
|
||||
model_group: Final = request.get("model")
|
||||
model_name: Final = model_group if isinstance(model_group, str) else None
|
||||
try:
|
||||
self._explain_fallbacks(error, model_name)
|
||||
except Exception: # noqa: BLE001 # Python looks the chains up inside its fallback try: a bad chain means none was attempted
|
||||
error.message += format_fallback_outcome_message( # pyright: ignore[reportAttributeAccessIssue] # checked above # rebind-ok: the error Python raises carries the same suffix
|
||||
model_name, None, "", skipped=self._walk.skipped
|
||||
)
|
||||
|
||||
def _explain_fallbacks(self, error: BaseException, model_name: str | None) -> None:
|
||||
import litellm.exceptions as litellm_exceptions
|
||||
|
||||
python: Final = self._python
|
||||
request: Final = self._request
|
||||
fallbacks: Final = python._core_fallbacks
|
||||
context_window_fallbacks: Final = python._core_context_window_fallbacks
|
||||
content_policy_fallbacks: Final = python._core_content_policy_fallbacks
|
||||
lookup_groups: Final = fallback_lookup_groups(request, model_name)
|
||||
context_window: Final = isinstance(error, litellm_exceptions.ContextWindowExceededError)
|
||||
content_policy: Final = isinstance(error, litellm_exceptions.ContentPolicyViolationError)
|
||||
attempted_group: list[object] | None = None # mutable-ok: lookup's list # rebind-ok: set below
|
||||
orders: Final = sorted(
|
||||
{
|
||||
order
|
||||
for deployment in python._core_get_model_list(model_name=model_name) or ()
|
||||
if (order := (deployment.get("litellm_params") or {}).get("order")) is not None
|
||||
}
|
||||
)
|
||||
if len(orders) > 1 and not (context_window or content_policy):
|
||||
external, generic = (
|
||||
get_fallback_model_group_for_lookup_groups(fallbacks=fallbacks, lookup_groups=lookup_groups)
|
||||
if fallbacks is not None and lookup_groups
|
||||
else (None, None)
|
||||
)
|
||||
if external is None and generic is not None and fallbacks is not None:
|
||||
external = fallbacks[generic]["*"] # rebind-ok: the generic chain when no specific one matched
|
||||
combined: Final = [{"model": model_name, "_target_order": o} for o in orders if o > orders[0]] + list(
|
||||
external or ()
|
||||
)
|
||||
if combined:
|
||||
attempted_group = combined # rebind-ok: settled in the order Python's walk does
|
||||
if attempted_group is None:
|
||||
specific: Final = (
|
||||
context_window_fallbacks if context_window else content_policy_fallbacks if content_policy else None
|
||||
)
|
||||
if (context_window or content_policy) and specific is not None:
|
||||
attempted_group, _ = ( # rebind-ok: settled in the order Python's walk does
|
||||
get_fallback_model_group_for_lookup_groups(fallbacks=specific, lookup_groups=lookup_groups)
|
||||
)
|
||||
if attempted_group is None:
|
||||
# Python raises before its generic fallbacks: "No fallback was attempted".
|
||||
error.message += format_fallback_outcome_message( # pyright: ignore[reportAttributeAccessIssue] # checked above # rebind-ok: the error Python raises carries the same suffix
|
||||
model_name, None, "", skipped=self._walk.skipped
|
||||
)
|
||||
return
|
||||
elif context_window:
|
||||
error.message += ( # pyright: ignore[reportAttributeAccessIssue] # checked above # rebind-ok: the error Python raises carries the same suffix
|
||||
f"\nmodel={model_name}. context_window_fallbacks={mask_sensitive_structure(context_window_fallbacks)}."
|
||||
f" fallbacks={mask_sensitive_structure(fallbacks)}.\n\nSet 'context_window_fallback' -"
|
||||
" https://docs.litellm.ai/docs/routing#fallbacks"
|
||||
)
|
||||
elif content_policy:
|
||||
error.message += ( # rebind-ok: the error Python raises carries the same suffix
|
||||
f"\n{format_content_policy_fallback_hint(model_name, content_policy_fallbacks, fallbacks)}" # pyright: ignore[reportAttributeAccessIssue] # checked above
|
||||
)
|
||||
if attempted_group is None and fallbacks is not None and lookup_groups:
|
||||
attempted_group, generic_index = ( # rebind-ok: settled in the order Python's walk does
|
||||
get_fallback_model_group_for_lookup_groups(fallbacks=fallbacks, lookup_groups=lookup_groups)
|
||||
)
|
||||
if attempted_group is None and generic_index is not None:
|
||||
attempted_group = fallbacks[generic_index][ # rebind-ok: settled in the order Python's walk does
|
||||
"*"
|
||||
]
|
||||
if attempted_group is None:
|
||||
error.message += format_no_fallback_group_message(lookup_groups, fallbacks) # pyright: ignore[reportAttributeAccessIssue] # checked above # rebind-ok: the error Python raises carries the same suffix
|
||||
return
|
||||
detail: Final = (
|
||||
truncate_fallback_error_detail(redact_string(str(self._level_errors.get(1) or error)))
|
||||
if attempted_group
|
||||
else ""
|
||||
)
|
||||
error.message += format_fallback_outcome_message( # pyright: ignore[reportAttributeAccessIssue] # checked above # rebind-ok: the error Python raises carries the same suffix
|
||||
model_name, attempted_group, detail, skipped=self._walk.skipped
|
||||
)
|
||||
|
||||
def _pinned_request(self, step: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""The request as the Python router's own fallback walk would send it, narrowed to the one
|
||||
deployment Rust picked: the name Rust resolved, with every other deployment it resolved to
|
||||
skipped."""
|
||||
deployment_id: Final = str(step["deployment_id"])
|
||||
name: Final = str(step.get("model") or step["model_group"])
|
||||
candidates: Final = step.get("candidate_ids")
|
||||
others: Final = tuple(
|
||||
str(i) for i in (candidates if isinstance(candidates, list) else ()) if str(i) != deployment_id
|
||||
)
|
||||
pinned: Final[dict[str, object]] = { # mutable-ok: built here, then handed to the Python core as its request
|
||||
**self._request,
|
||||
"model": name,
|
||||
}
|
||||
if others:
|
||||
pinned["_retry_skipped_deployment_ids"] = others
|
||||
target_order: Final = step.get("target_order")
|
||||
if isinstance(target_order, int):
|
||||
pinned["_target_order"] = target_order
|
||||
return pinned
|
||||
|
||||
async def _attempt(self, step: Mapping[str, object]) -> Mapping[str, object]:
|
||||
self._attempted = True
|
||||
self._attempted_groups.add(str(step["model_group"]))
|
||||
self._timeline.append(("attempt", step.get("fallback_level")))
|
||||
request: Final = self._pinned_request(step)
|
||||
try:
|
||||
response: Final = await self._python.call(
|
||||
self._op, {**request, "num_retries": 0, "disable_fallbacks": True}
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # every attempt failure is classified and handed to the Rust call loop
|
||||
self._last_error = error
|
||||
level: Final = step.get("fallback_level")
|
||||
self._level_errors[level if isinstance(level, int) else 0] = error
|
||||
self._timeline.append(("error", (level if isinstance(level, int) else 0, error, request)))
|
||||
return attempt_failure(error)
|
||||
self._last_response = response
|
||||
return {"ok": True, "total_tokens": total_tokens(response)}
|
||||
|
||||
async def _allow_fallback(self, target: str) -> bool:
|
||||
"""`_is_fallback_target_authorized` then `_is_fallback_target_within_budget`."""
|
||||
ctx: Final = self._python._core_ctx
|
||||
settings: Final = ctx.settings()
|
||||
request: Final = dict(self._request)
|
||||
original: Final = self._request.get("model")
|
||||
if target == original:
|
||||
return True
|
||||
if settings.fallback_access_check is not None and not await ctx.fallback_access_allowed(
|
||||
model=target, request_kwargs=request
|
||||
):
|
||||
self._walk.record_skip(target, FallbackSkipReason.NOT_AUTHORIZED)
|
||||
return False
|
||||
if settings.fallback_budget_check is not None and not await ctx.fallback_budget_allowed(
|
||||
model=target, request_kwargs=request
|
||||
):
|
||||
self._walk.record_skip(target, FallbackSkipReason.OVER_BUDGET)
|
||||
return False
|
||||
self._hop_targets.append(target)
|
||||
return True
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from asyncio import Future
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Never, final
|
||||
from typing import Literal, Never, NotRequired, TypedDict, final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue
|
||||
|
|
@ -218,6 +218,8 @@ __all__ = [
|
|||
"NativeDiagnosticProcessor",
|
||||
"ProcessReservedForForking",
|
||||
"ResponsesWebSocketConnection",
|
||||
"RouterCall",
|
||||
"RouterCoreHandle",
|
||||
"RustBridgeDeclined",
|
||||
"RustUpstreamError",
|
||||
"TokenCounter",
|
||||
|
|
@ -325,3 +327,115 @@ class NativeCacheHandle:
|
|||
def ping(self) -> Future[bool]: ...
|
||||
def disconnect(self) -> Future[None]: ...
|
||||
def delete(self, keys: Sequence[str]) -> Future[None]: ...
|
||||
|
||||
# The FailureKind names an attempt outcome's `kind` takes (router-core `FailureKind`).
|
||||
_FailureKind = Literal[
|
||||
"RateLimitError",
|
||||
"ContextWindowExceededError",
|
||||
"ContentPolicyViolationError",
|
||||
"AuthenticationError",
|
||||
"PermissionDeniedError",
|
||||
"NotFoundError",
|
||||
"BadRequestError",
|
||||
"Timeout",
|
||||
"APIConnectionError",
|
||||
"InternalServerError",
|
||||
"ServiceUnavailableError",
|
||||
"GuardrailIntervention",
|
||||
"Declined",
|
||||
"CallerCredentialError",
|
||||
"APIError",
|
||||
]
|
||||
|
||||
class _AttemptStep(TypedDict):
|
||||
kind: Literal["attempt"]
|
||||
deployment_id: str
|
||||
model_group: str
|
||||
retry: int
|
||||
fallback_depth: int
|
||||
timeout: float | None
|
||||
|
||||
class _SleepStep(TypedDict):
|
||||
kind: Literal["sleep"]
|
||||
seconds: float
|
||||
|
||||
class _AllowFallbackStep(TypedDict):
|
||||
kind: Literal["allow_fallback"]
|
||||
target: str
|
||||
|
||||
class _SelectStep(TypedDict):
|
||||
kind: Literal["select"]
|
||||
stage: Literal["plugins", "strategy"]
|
||||
strategy: str | None
|
||||
model_group: str
|
||||
candidates: list[str]
|
||||
|
||||
# One RouterCache op against the router's DualCache. `key` is set for every op but `batch_get`,
|
||||
# which sets `keys`; `value` is the JSON to store (`set`) or the amount (`increment`); `ttl` is
|
||||
# in seconds. Outcomes: get -> the JSON value or None; batch_get -> one JSON value or None per
|
||||
# key, in order; increment -> the new int or float; set and delete -> None.
|
||||
class _CacheStep(TypedDict):
|
||||
kind: Literal["cache"]
|
||||
op: Literal["get", "set", "increment", "batch_get", "delete"]
|
||||
key: str | None
|
||||
keys: list[str] | None
|
||||
value: JsonValue
|
||||
ttl: float | None
|
||||
|
||||
class _DoneStep(TypedDict):
|
||||
kind: Literal["done"]
|
||||
deployment_id: str
|
||||
model_group: str
|
||||
attempted_retries: int
|
||||
attempted_fallbacks: int
|
||||
total_tokens: int
|
||||
|
||||
class _FailedStep(TypedDict):
|
||||
kind: Literal["failed"]
|
||||
error: str
|
||||
error_kind: Literal["config", "unknown_model", "no_healthy_deployments", "attempt", "cache", "retired"]
|
||||
attempt_failed: bool
|
||||
status: int | None
|
||||
failure_kind: _FailureKind | None
|
||||
|
||||
# The end of a `RouterCoreHandle.select`.
|
||||
class _SelectedStep(TypedDict):
|
||||
kind: Literal["done"]
|
||||
deployment_id: str
|
||||
model_group: str
|
||||
|
||||
_RouterStep = _AttemptStep | _SleepStep | _AllowFallbackStep | _SelectStep | _CacheStep | _DoneStep | _FailedStep
|
||||
|
||||
class _AttemptSucceeded(TypedDict):
|
||||
ok: Literal[True]
|
||||
total_tokens: NotRequired[int]
|
||||
|
||||
class _AttemptFailed(TypedDict):
|
||||
ok: Literal[False]
|
||||
kind: _FailureKind
|
||||
status: NotRequired[int | None]
|
||||
retry_after: NotRequired[float | None]
|
||||
num_retries: NotRequired[int | None]
|
||||
detail: NotRequired[str | None]
|
||||
|
||||
# Each document is a mapping or its JSON text.
|
||||
_Document = Mapping[str, object] | str
|
||||
|
||||
@final
|
||||
class RouterCoreHandle:
|
||||
def __new__(cls, config: _Document, dynamic: _Document) -> RouterCoreHandle: ...
|
||||
def start(self, op: str, request: _Document) -> RouterCall: ...
|
||||
# Driven like `start`; its `done` step is a `_SelectedStep`.
|
||||
def select(self, model: str, request: _Document) -> RouterCall: ...
|
||||
def apply(self, changes: _Document) -> int: ...
|
||||
def registry_version(self) -> int: ...
|
||||
def retire(self) -> None: ...
|
||||
|
||||
@final
|
||||
class RouterCall:
|
||||
# attempt: _AttemptSucceeded | _AttemptFailed; sleep: None; allow_fallback: bool;
|
||||
# select: the remaining candidate ids, pick first; cache: see _CacheStep; first call: None.
|
||||
# A call from `select` steps through select and cache only, and ends in _SelectedStep or failed.
|
||||
def next(
|
||||
self, outcome: _AttemptSucceeded | _AttemptFailed | bool | Sequence[str] | JsonValue | None
|
||||
) -> _RouterStep | _SelectedStep: ...
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ class Route(str, Enum):
|
|||
OCR = "ocr"
|
||||
TOKEN_COUNTER = "token_counter"
|
||||
TOKENIZER = "tokenizer"
|
||||
ROUTER = "router"
|
||||
|
||||
|
||||
class Target(str, Enum):
|
||||
|
|
@ -221,6 +222,7 @@ RULES: Final[Rules] = (
|
|||
RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY),
|
||||
RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY),
|
||||
RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY),
|
||||
RouteRule(Route.ROUTER, Rollout.RUST_OPT_IN),
|
||||
RouteRule(Route.TRANSCRIPTION, Rollout.RUST_REQUIRED, providers=frozenset({"bedrock"})),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ CHECKED = (
|
|||
"litellm/router_core.py",
|
||||
"litellm/router_deprecated.py",
|
||||
"litellm/router_python_core.py",
|
||||
"litellm/router_rust_core.py",
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ ROUTER_FILES = {
|
|||
"litellm/router_core.py",
|
||||
"litellm/router_python_core.py",
|
||||
"litellm/router_python_host.py",
|
||||
"litellm/router_rust_core.py",
|
||||
"litellm/router_deprecated.py",
|
||||
"litellm/router_host.py",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -452,25 +452,36 @@ def python_core(host: ScriptedHost, **arguments: object) -> PythonCore: # kwarg
|
|||
return PythonCore(**config.arguments, _host=host) # pyright: ignore[reportArgumentType] # the same arguments Router binds
|
||||
|
||||
|
||||
CoreBuilder = Callable[..., RouterCore]
|
||||
"""``builder(host, **router_arguments)``: a router core on ``host``."""
|
||||
|
||||
CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core}
|
||||
"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a
|
||||
configuration raises ``CoreDeclined``, and the test is skipped for that core."""
|
||||
|
||||
|
||||
class CoreDeclined(Exception):
|
||||
"""The core does not serve this configuration (the router would stay on the Python core)."""
|
||||
|
||||
|
||||
CoreBuilder = Callable[..., RouterCore]
|
||||
"""``builder(host, **router_arguments)``: a router core on ``host``."""
|
||||
|
||||
|
||||
def rust_core(host: ScriptedHost, **arguments: object) -> RouterCore: # kwargs-ok: the router's constructor arguments
|
||||
"""A ``RustCore`` on ``host``, built the way ``make_core`` builds one when the Rust router is on."""
|
||||
from litellm.router_rust_core import RustCore
|
||||
|
||||
try:
|
||||
return RustCore.build(build_router_config(arguments), host)
|
||||
except Exception as declined: # noqa: BLE001 # make_core keeps such a router on the Python core
|
||||
raise CoreDeclined(str(declined)) from declined
|
||||
|
||||
|
||||
CORE_BUILDERS: Final[dict[str, CoreBuilder]] = {"python": python_core, "rust": rust_core}
|
||||
"""Every core the suite runs on. Another core adds its builder here; a builder that cannot take a
|
||||
configuration raises ``CoreDeclined``, and the test is skipped for that core."""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Call",
|
||||
"HostCallbacks",
|
||||
"INFERENCE_OPS",
|
||||
"CORE_BUILDERS",
|
||||
"INFERENCE_OPS",
|
||||
"Call",
|
||||
"CoreBuilder",
|
||||
"CoreDeclined",
|
||||
"HostCallbacks",
|
||||
"ScriptedHost",
|
||||
"auth_error",
|
||||
"bad_request",
|
||||
|
|
@ -482,6 +493,7 @@ __all__ = [
|
|||
"embedding_response",
|
||||
"python_core",
|
||||
"rate_limit",
|
||||
"rust_core",
|
||||
"server_error",
|
||||
"timeout",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -622,3 +622,40 @@ async def test_cooling_down_one_group_leaves_other_groups_pickable(make_core: Ma
|
|||
with pytest.raises(RouterRateLimitError):
|
||||
await core.select("g")
|
||||
assert await _picked_ids(core, "h") <= {"c", "d"}
|
||||
|
||||
|
||||
# --- custom strategies ---------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_registered_strategy_picks_every_deployment_an_endpoint_call_reaches(
|
||||
make_core: MakeCore, host: ScriptedHost
|
||||
) -> None:
|
||||
from litellm.types.router import CustomRoutingStrategyBase
|
||||
|
||||
core: Final = make_core(model_list=[deployment("g", "a"), deployment("g", "b"), deployment("g", "c")])
|
||||
seen: Final[list[Mapping[str, object]]] = [] # mutable-ok: the request each pick was asked for
|
||||
|
||||
class HighestId(CustomRoutingStrategyBase):
|
||||
async def async_get_available_deployment(
|
||||
self,
|
||||
model: str,
|
||||
messages: list[dict[str, str]] | None = None, # mutable-ok: the strategy interface's own type
|
||||
input: str | list | None = None, # mutable-ok: the strategy interface's own type
|
||||
specific_deployment: bool | None = False,
|
||||
request_kwargs: dict | None = None, # mutable-ok: the strategy interface's own type
|
||||
) -> dict: # mutable-ok: the strategy interface's own type
|
||||
seen.append(dict(request_kwargs or {}))
|
||||
deployments: Final = core.get_model_list(model_name=model) # pyright: ignore[reportAttributeAccessIssue] # both cores serve the Python core's lookup
|
||||
return max(deployments, key=lambda d: d["model_info"]["id"])
|
||||
|
||||
def get_available_deployment(self, model: str) -> dict: # mutable-ok: the strategy interface's own type
|
||||
raise NotImplementedError
|
||||
|
||||
core.register_plugin(HighestId())
|
||||
for _ in range(PICKS):
|
||||
await core.acompletion(model="g", messages=MESSAGES) # pyright: ignore[reportAttributeAccessIssue] # endpoint method
|
||||
assert set(host.deployments_called("acompletion")) == {"c"}
|
||||
# The strategy picks from the whole group: no other core narrowed the pick before it.
|
||||
assert len(seen) == PICKS
|
||||
assert not any("_retry_skipped_deployment_ids" in request for request in seen)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue