mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
feat(rust): add circuit-breaker crate and guard redis-client with it
This commit is contained in:
parent
7b58981288
commit
f8920b5403
14 changed files with 841 additions and 7 deletions
9
litellm-rust/Cargo.lock
generated
9
litellm-rust/Cargo.lock
generated
|
|
@ -3655,6 +3655,14 @@ dependencies = [
|
|||
"strum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-circuit-breaker"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"proptest",
|
||||
"rstest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-config"
|
||||
version = "0.1.0"
|
||||
|
|
@ -4293,6 +4301,7 @@ name = "litellm-redis-client"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"litellm-circuit-breaker",
|
||||
"litellm-redis-testing",
|
||||
"redis",
|
||||
"rstest",
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface"
|
|||
litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" }
|
||||
litellm-host-python = { path = "crates/host-python" }
|
||||
litellm-python-compat = { path = "crates/python-compat" }
|
||||
litellm-circuit-breaker = { path = "crates/circuit-breaker" }
|
||||
litellm-redis-client = { path = "crates/redis-client" }
|
||||
litellm-redis-testing = { path = "crates/redis-testing" }
|
||||
|
||||
|
|
|
|||
14
litellm-rust/crates/circuit-breaker/AGENTS.md
Normal file
14
litellm-rust/crates/circuit-breaker/AGENTS.md
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
- `fsm` is a generic state machine runtime and knows nothing about breakers. `breaker` is a generic circuit breaker and knows nothing about Redis. Deciding which errors count as failures belongs to the consumer
|
||||
- `Machine::step` stays pure. `&self` is config and time comes in on the event, never read inside the machine. The only clock read is `Admission`'s drop, at the call-site boundary
|
||||
- Transition rules are one `match (state, event)` per machine, with no macro DSL and no `_` arm that would hide a new state from the compiler
|
||||
- `Shared` counts a transition, bumps the generation and calls the observer only when the state's enum variant changes, matching Python bumping `generation` only when `_state` changes. Data inside a variant (failure counters, `since`) changes without a transition
|
||||
- The observer runs under the lock so transitions are seen in generation order. Keep it to logging or a metric
|
||||
- A disabled breaker is `Option<Breaker>::None` at the consumer, not a flag inside the machine. `Config` comes in already resolved, reading `REDIS_CIRCUIT_BREAKER_*` env is the consumer's job
|
||||
|
||||
## Parity with Python `RedisCircuitBreaker` (`litellm/caching/redis_cache.py`)
|
||||
- The rstest table `matches_the_python_redis_circuit_breaker` was checked case by case against the Python breaker. A behavior change here needs the same check, by driving the Python class with the same steps
|
||||
- Divergences, each on purpose
|
||||
- monotonic `Instant` instead of `time.time()`, so a wall clock jump can't open or close the breaker
|
||||
- a probe `Admission` dropped without an outcome records a hard failure and reopens. Python's probe that dies on `CancelledError` (a `BaseException`) records nothing and leaves the breaker HALF_OPEN for good, and a dropped Rust future is common
|
||||
- `_failure_count` is dropped, it only fed a log line
|
||||
- no `enabled` flag, see above
|
||||
12
litellm-rust/crates/circuit-breaker/Cargo.toml
Normal file
12
litellm-rust/crates/circuit-breaker/Cargo.toml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
[package]
|
||||
name = "litellm-circuit-breaker"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
|
||||
[dev-dependencies]
|
||||
proptest.workspace = true
|
||||
rstest.workspace = true
|
||||
414
litellm-rust/crates/circuit-breaker/src/breaker.rs
Normal file
414
litellm-rust/crates/circuit-breaker/src/breaker.rs
Normal file
|
|
@ -0,0 +1,414 @@
|
|||
use std::{
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use crate::{Machine, Observer, Shared};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct Config {
|
||||
pub failure_threshold: u32,
|
||||
pub recovery_timeout: Duration,
|
||||
pub timeout_min_duration: Duration,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
failure_threshold: 5,
|
||||
recovery_timeout: Duration::from_secs(60),
|
||||
timeout_min_duration: Duration::from_secs(5),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Failure {
|
||||
Hard,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum State {
|
||||
Closed {
|
||||
hard: u32,
|
||||
timeouts: u32,
|
||||
streak_started: Option<Instant>,
|
||||
},
|
||||
Open {
|
||||
since: Instant,
|
||||
},
|
||||
HalfOpen,
|
||||
}
|
||||
|
||||
impl State {
|
||||
const CLOSED: Self = Self::Closed {
|
||||
hard: 0,
|
||||
timeouts: 0,
|
||||
streak_started: None,
|
||||
};
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum Event {
|
||||
Admit(Instant),
|
||||
Succeeded,
|
||||
Failed(Failure, Instant),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum Output {
|
||||
Admitted,
|
||||
Probe,
|
||||
Rejected,
|
||||
Recorded,
|
||||
}
|
||||
|
||||
impl Machine for Config {
|
||||
type State = State;
|
||||
type Event = Event;
|
||||
type Output = Output;
|
||||
|
||||
fn step(&self, state: &State, event: Event) -> (State, Output) {
|
||||
match (state, event) {
|
||||
(State::Closed { .. }, Event::Admit(_)) => (state.clone(), Output::Admitted),
|
||||
(State::Open { since }, Event::Admit(now))
|
||||
if now.duration_since(*since) > self.recovery_timeout =>
|
||||
{
|
||||
(State::HalfOpen, Output::Probe)
|
||||
}
|
||||
(State::Open { .. } | State::HalfOpen, Event::Admit(_)) => {
|
||||
(state.clone(), Output::Rejected)
|
||||
}
|
||||
(State::Closed { .. } | State::HalfOpen, Event::Succeeded) => {
|
||||
(State::CLOSED, Output::Recorded)
|
||||
}
|
||||
(State::Open { .. }, Event::Succeeded) => (state.clone(), Output::Recorded),
|
||||
(State::Closed { hard, .. }, Event::Failed(Failure::Hard, now)) => {
|
||||
let hard = hard + 1;
|
||||
let next = match hard >= self.failure_threshold {
|
||||
true => State::Open { since: now },
|
||||
false => State::Closed {
|
||||
hard,
|
||||
timeouts: 0,
|
||||
streak_started: None,
|
||||
},
|
||||
};
|
||||
(next, Output::Recorded)
|
||||
}
|
||||
(
|
||||
State::Closed {
|
||||
hard,
|
||||
timeouts,
|
||||
streak_started,
|
||||
},
|
||||
Event::Failed(Failure::Timeout, now),
|
||||
) => {
|
||||
let timeouts = timeouts + 1;
|
||||
let started = streak_started.unwrap_or(now);
|
||||
let opens = timeouts >= self.failure_threshold
|
||||
&& now.duration_since(started) >= self.timeout_min_duration;
|
||||
let next = match opens {
|
||||
true => State::Open { since: now },
|
||||
false => State::Closed {
|
||||
hard: *hard,
|
||||
timeouts,
|
||||
streak_started: Some(started),
|
||||
},
|
||||
};
|
||||
(next, Output::Recorded)
|
||||
}
|
||||
(State::Open { .. } | State::HalfOpen, Event::Failed(_, now)) => {
|
||||
(State::Open { since: now }, Output::Recorded)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Breaker(Arc<Shared<Config>>);
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Rejected;
|
||||
|
||||
pub struct Admission {
|
||||
breaker: Breaker,
|
||||
generation: u64,
|
||||
probe: bool,
|
||||
settled: bool,
|
||||
}
|
||||
|
||||
impl Breaker {
|
||||
pub fn new(config: Config, observer: Observer<State>) -> Self {
|
||||
Self(Arc::new(Shared::new(config, State::CLOSED, observer)))
|
||||
}
|
||||
|
||||
pub fn admit(&self, now: Instant) -> Result<Admission, Rejected> {
|
||||
let (output, generation) = self.0.fire(Event::Admit(now));
|
||||
let probe = match output {
|
||||
Output::Admitted => false,
|
||||
Output::Probe => true,
|
||||
Output::Rejected | Output::Recorded => return Err(Rejected),
|
||||
};
|
||||
Ok(Admission {
|
||||
breaker: self.clone(),
|
||||
generation,
|
||||
probe,
|
||||
settled: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn failed(&self, failure: Failure, now: Instant) {
|
||||
self.0.fire(Event::Failed(failure, now));
|
||||
}
|
||||
|
||||
pub fn state(&self) -> State {
|
||||
self.0.state()
|
||||
}
|
||||
}
|
||||
|
||||
impl Admission {
|
||||
pub fn probe(&self) -> bool {
|
||||
self.probe
|
||||
}
|
||||
|
||||
pub fn succeeded(mut self) {
|
||||
self.settled = true;
|
||||
self.breaker.0.fire_if(self.generation, Event::Succeeded);
|
||||
}
|
||||
|
||||
pub fn failed(mut self, failure: Failure, now: Instant) {
|
||||
self.settled = true;
|
||||
self.breaker.failed(failure, now);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Admission {
|
||||
fn drop(&mut self) {
|
||||
if self.probe && !self.settled {
|
||||
self.breaker.failed(Failure::Hard, Instant::now());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use proptest::prelude::*;
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{Admission, Breaker, Config, Failure, State};
|
||||
use crate::Transition;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Step {
|
||||
Fail(f64, Failure),
|
||||
Admit(f64, bool),
|
||||
Succeed(usize),
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Phase {
|
||||
Closed,
|
||||
Open,
|
||||
HalfOpen,
|
||||
}
|
||||
|
||||
use Phase::{Closed as C, HalfOpen as H, Open as O};
|
||||
|
||||
fn hard(at: f64) -> Step {
|
||||
Step::Fail(at, Failure::Hard)
|
||||
}
|
||||
|
||||
fn timeout(at: f64) -> Step {
|
||||
Step::Fail(at, Failure::Timeout)
|
||||
}
|
||||
|
||||
fn five_hard_at_zero() -> Vec<Step> {
|
||||
vec![hard(0.0); 5]
|
||||
}
|
||||
|
||||
fn phase(state: &State) -> Phase {
|
||||
match state {
|
||||
State::Closed { .. } => Phase::Closed,
|
||||
State::Open { .. } => Phase::Open,
|
||||
State::HalfOpen => Phase::HalfOpen,
|
||||
}
|
||||
}
|
||||
|
||||
fn quiet() -> Breaker {
|
||||
Breaker::new(Config::default(), Box::new(|_| {}))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::five_hard_failures_open(vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), hard(4.0)], vec![C, C, C, C, O])]
|
||||
#[case::a_success_resets_the_hard_failure_count(
|
||||
vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), Step::Admit(4.0, true), Step::Succeed(4), hard(5.0), hard(6.0), hard(7.0), hard(8.0)],
|
||||
vec![C; 10]
|
||||
)]
|
||||
#[case::a_timeout_streak_opens_only_after_the_min_duration(
|
||||
vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), timeout(4.9), timeout(5.0)],
|
||||
vec![C, C, C, C, C, O]
|
||||
)]
|
||||
#[case::a_hard_failure_restarts_the_timeout_streak(
|
||||
vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), hard(4.0), timeout(5.0), timeout(6.0), timeout(7.0), timeout(8.0), timeout(9.9), timeout(10.0)],
|
||||
vec![C, C, C, C, C, C, C, C, C, C, O]
|
||||
)]
|
||||
#[case::the_probe_waits_until_strictly_past_the_recovery_timeout(
|
||||
[five_hard_at_zero(), vec![Step::Admit(60.0, false), Step::Admit(60.5, true)]].concat(),
|
||||
vec![C, C, C, C, O, O, H]
|
||||
)]
|
||||
#[case::a_probe_success_closes(
|
||||
[five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Succeed(5)]].concat(),
|
||||
vec![C, C, C, C, O, H, C]
|
||||
)]
|
||||
#[case::a_probe_failure_reopens_with_a_fresh_timer(
|
||||
[five_hard_at_zero(), vec![Step::Admit(61.0, true), hard(62.0), Step::Admit(122.0, false), Step::Admit(122.5, true)]].concat(),
|
||||
vec![C, C, C, C, O, H, O, O, H]
|
||||
)]
|
||||
#[case::only_one_probe_at_a_time(
|
||||
[five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Admit(61.5, false)]].concat(),
|
||||
vec![C, C, C, C, O, H, H]
|
||||
)]
|
||||
#[case::a_late_failure_while_open_pushes_back_the_probe(
|
||||
[five_hard_at_zero(), vec![hard(30.0), Step::Admit(61.0, false), Step::Admit(90.5, true)]].concat(),
|
||||
vec![C, C, C, C, O, O, O, H]
|
||||
)]
|
||||
#[case::a_success_from_before_the_breaker_moved_is_ignored(
|
||||
vec![Step::Admit(0.0, true), hard(1.0), hard(1.0), hard(1.0), hard(1.0), hard(1.0), Step::Admit(62.0, true), Step::Succeed(0), Step::Succeed(6)],
|
||||
vec![C, C, C, C, C, O, H, H, C]
|
||||
)]
|
||||
fn matches_the_python_redis_circuit_breaker(
|
||||
#[case] steps: Vec<Step>,
|
||||
#[case] expected: Vec<Phase>,
|
||||
) {
|
||||
let breaker = quiet();
|
||||
let t0 = Instant::now();
|
||||
let mut admissions: HashMap<usize, Admission> = HashMap::new();
|
||||
|
||||
let phases: Vec<Phase> = steps
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, step)| {
|
||||
match *step {
|
||||
Step::Fail(at, failure) => {
|
||||
breaker.failed(failure, t0 + Duration::from_secs_f64(at))
|
||||
}
|
||||
Step::Admit(at, admitted) => {
|
||||
let admission = breaker.admit(t0 + Duration::from_secs_f64(at));
|
||||
assert_eq!(admission.is_ok(), admitted, "step {index}");
|
||||
admissions.extend(admission.ok().map(|admission| (index, admission)));
|
||||
}
|
||||
Step::Succeed(admitted_at) => {
|
||||
admissions.remove(&admitted_at).unwrap().succeeded()
|
||||
}
|
||||
}
|
||||
phase(&breaker.state())
|
||||
})
|
||||
.collect();
|
||||
|
||||
assert_eq!(phases, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_dropped_probe_reopens_instead_of_wedging_half_open() {
|
||||
let breaker = quiet();
|
||||
let t0 = Instant::now();
|
||||
(0..5).for_each(|_| breaker.failed(Failure::Hard, t0));
|
||||
let probe = breaker.admit(t0 + Duration::from_secs(61)).unwrap();
|
||||
assert!(probe.probe());
|
||||
|
||||
drop(probe);
|
||||
|
||||
assert!(matches!(breaker.state(), State::Open { .. }));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_dropped_regular_call_records_nothing() {
|
||||
let breaker = quiet();
|
||||
let t0 = Instant::now();
|
||||
breaker.failed(Failure::Hard, t0);
|
||||
|
||||
drop(breaker.admit(t0).unwrap());
|
||||
|
||||
assert_eq!(
|
||||
breaker.state(),
|
||||
State::Closed {
|
||||
hard: 1,
|
||||
timeouts: 0,
|
||||
streak_started: None
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Action {
|
||||
Admit,
|
||||
SucceedOldest,
|
||||
FailOldest(Failure),
|
||||
Fail(Failure),
|
||||
}
|
||||
|
||||
fn action() -> impl Strategy<Value = (u64, Action)> {
|
||||
let failure = prop_oneof![Just(Failure::Hard), Just(Failure::Timeout)];
|
||||
let action = prop_oneof![
|
||||
Just(Action::Admit),
|
||||
Just(Action::SucceedOldest),
|
||||
failure.clone().prop_map(Action::FailOldest),
|
||||
failure.prop_map(Action::Fail),
|
||||
];
|
||||
(0u64..30_000, action)
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn open_never_closes_directly_and_each_half_open_visit_has_one_probe(
|
||||
actions in prop::collection::vec(action(), 1..200)
|
||||
) {
|
||||
let transitions: Arc<Mutex<Vec<Transition<State>>>> = Arc::default();
|
||||
let seen = Arc::clone(&transitions);
|
||||
let config = Config {
|
||||
failure_threshold: 3,
|
||||
recovery_timeout: Duration::from_secs(10),
|
||||
timeout_min_duration: Duration::from_secs(2),
|
||||
};
|
||||
let breaker = Breaker::new(config, Box::new(move |t| seen.lock().unwrap().push(t.clone())));
|
||||
let t0 = Instant::now();
|
||||
let mut now = t0;
|
||||
let mut pending: Vec<Admission> = Vec::new();
|
||||
let mut probes_per_visit: Vec<u32> = Vec::new();
|
||||
|
||||
for (advance_ms, action) in actions {
|
||||
now += Duration::from_millis(advance_ms);
|
||||
match action {
|
||||
Action::Admit => {
|
||||
if let Ok(admission) = breaker.admit(now) {
|
||||
if admission.probe() {
|
||||
probes_per_visit.push(1);
|
||||
}
|
||||
pending.push(admission);
|
||||
}
|
||||
}
|
||||
Action::SucceedOldest if !pending.is_empty() => pending.remove(0).succeeded(),
|
||||
Action::FailOldest(failure) if !pending.is_empty() => pending.remove(0).failed(failure, now),
|
||||
Action::SucceedOldest | Action::FailOldest(_) => {}
|
||||
Action::Fail(failure) => breaker.failed(failure, now),
|
||||
}
|
||||
}
|
||||
|
||||
let transitions = transitions.lock().unwrap();
|
||||
let open_to_closed = transitions
|
||||
.iter()
|
||||
.filter(|t| matches!((&t.from, &t.to), (State::Open { .. }, State::Closed { .. })))
|
||||
.count();
|
||||
let half_open_visits = transitions.iter().filter(|t| t.to == State::HalfOpen).count();
|
||||
prop_assert_eq!(open_to_closed, 0);
|
||||
prop_assert_eq!(probes_per_visit.len(), half_open_visits);
|
||||
}
|
||||
}
|
||||
}
|
||||
174
litellm-rust/crates/circuit-breaker/src/fsm.rs
Normal file
174
litellm-rust/crates/circuit-breaker/src/fsm.rs
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
use std::{
|
||||
fmt::Debug,
|
||||
mem,
|
||||
sync::{Mutex, PoisonError},
|
||||
};
|
||||
|
||||
pub trait Machine {
|
||||
type State: Clone + Debug;
|
||||
type Event;
|
||||
type Output;
|
||||
|
||||
fn step(&self, state: &Self::State, event: Self::Event) -> (Self::State, Self::Output);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Transition<S> {
|
||||
pub from: S,
|
||||
pub to: S,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
pub type Observer<S> = Box<dyn Fn(&Transition<S>) + Send + Sync>;
|
||||
|
||||
pub struct Shared<M: Machine> {
|
||||
machine: M,
|
||||
current: Mutex<(M::State, u64)>,
|
||||
observer: Observer<M::State>,
|
||||
}
|
||||
|
||||
impl<M: Machine> Shared<M> {
|
||||
pub fn new(machine: M, initial: M::State, observer: Observer<M::State>) -> Self {
|
||||
Self {
|
||||
machine,
|
||||
current: Mutex::new((initial, 0)),
|
||||
observer,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn fire(&self, event: M::Event) -> (M::Output, u64) {
|
||||
let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
self.apply(&mut current, event)
|
||||
}
|
||||
|
||||
pub fn fire_if(&self, generation: u64, event: M::Event) -> Option<M::Output> {
|
||||
let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
(current.1 == generation).then(|| self.apply(&mut current, event).0)
|
||||
}
|
||||
|
||||
pub fn state(&self) -> M::State {
|
||||
self.current
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.0
|
||||
.clone()
|
||||
}
|
||||
|
||||
fn apply(&self, current: &mut (M::State, u64), event: M::Event) -> (M::Output, u64) {
|
||||
let (to, output) = self.machine.step(¤t.0, event);
|
||||
if mem::discriminant(&to) != mem::discriminant(¤t.0) {
|
||||
let from = mem::replace(&mut current.0, to.clone());
|
||||
current.1 += 1;
|
||||
(self.observer)(&Transition {
|
||||
from,
|
||||
to,
|
||||
generation: current.1,
|
||||
});
|
||||
} else {
|
||||
current.0 = to;
|
||||
}
|
||||
(output, current.1)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{Machine, Shared, Transition};
|
||||
|
||||
struct Wrapping(u32);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
enum Counter {
|
||||
Low(u32),
|
||||
High(u32),
|
||||
}
|
||||
|
||||
enum Tick {
|
||||
Up,
|
||||
Stay,
|
||||
}
|
||||
|
||||
impl Machine for Wrapping {
|
||||
type State = Counter;
|
||||
type Event = Tick;
|
||||
type Output = ();
|
||||
|
||||
fn step(&self, state: &Counter, event: Tick) -> (Counter, ()) {
|
||||
let value = match (state, event) {
|
||||
(Counter::Low(_) | Counter::High(_), Tick::Stay) => return (state.clone(), ()),
|
||||
(Counter::Low(value) | Counter::High(value), Tick::Up) => (value + 1) % self.0,
|
||||
};
|
||||
match value < self.0 / 2 {
|
||||
true => (Counter::Low(value), ()),
|
||||
false => (Counter::High(value), ()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type Seen = Arc<Mutex<Vec<Transition<Counter>>>>;
|
||||
|
||||
fn recorded(wrap: u32) -> (Shared<Wrapping>, Seen) {
|
||||
let transitions: Seen = Arc::default();
|
||||
let seen = Arc::clone(&transitions);
|
||||
let shared = Shared::new(
|
||||
Wrapping(wrap),
|
||||
Counter::Low(0),
|
||||
Box::new(move |transition| seen.lock().unwrap().push(transition.clone())),
|
||||
);
|
||||
(shared, transitions)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn concurrent_events_are_all_applied_and_transitions_are_observed_in_generation_order() {
|
||||
let (shared, transitions) = recorded(4);
|
||||
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..64 {
|
||||
scope.spawn(|| {
|
||||
for _ in 0..100 {
|
||||
shared.fire(Tick::Up);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
assert_eq!(shared.state(), Counter::Low(6400 % 4));
|
||||
let generations: Vec<u64> = transitions
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|t| t.generation)
|
||||
.collect();
|
||||
assert_eq!(generations, (1..=3200).collect::<Vec<u64>>());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn a_step_that_keeps_the_variant_neither_counts_nor_notifies() {
|
||||
let (shared, transitions) = recorded(4);
|
||||
|
||||
let (_, after_inner_change) = shared.fire(Tick::Up);
|
||||
let (_, after_self_loop) = shared.fire(Tick::Stay);
|
||||
|
||||
assert_eq!(shared.state(), Counter::Low(1));
|
||||
assert_eq!((after_inner_change, after_self_loop), (0, 0));
|
||||
assert!(transitions.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn fire_if_skips_an_event_from_an_older_generation() {
|
||||
let (shared, _) = recorded(4);
|
||||
shared.fire(Tick::Up);
|
||||
let (_, generation) = shared.fire(Tick::Up);
|
||||
|
||||
let stale = shared.fire_if(generation - 1, Tick::Up);
|
||||
let current = shared.fire_if(generation, Tick::Up);
|
||||
|
||||
assert!(stale.is_none());
|
||||
assert!(current.is_some());
|
||||
assert_eq!(shared.state(), Counter::High(3));
|
||||
}
|
||||
}
|
||||
5
litellm-rust/crates/circuit-breaker/src/lib.rs
Normal file
5
litellm-rust/crates/circuit-breaker/src/lib.rs
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
mod breaker;
|
||||
mod fsm;
|
||||
|
||||
pub use breaker::{Admission, Breaker, Config, Event, Failure, Output, Rejected, State};
|
||||
pub use fsm::{Machine, Observer, Shared, Transition};
|
||||
|
|
@ -12,6 +12,13 @@
|
|||
- Never swallow an error or turn it into a miss. The caller picks the fallback
|
||||
- Reads return `Option<T>`: a missing key is `None`, and a value that doesn't decode to `T` (`"15.0"` as `i64`) is `Decode`
|
||||
|
||||
## Circuit breaker
|
||||
- The `litellm-circuit-breaker` `Breaker` is injected through `RedisConfig::breaker` and shared by every clone. `None` turns it off
|
||||
- Every public call (each command, `eval`, one `run` of a pipeline) takes one admission through `guarded` / `admit` in `client.rs`. Never call the connection around it
|
||||
- `Transport` counts as a failure (`Timeout` when `is_timeout()`, else `Hard`). `Server` and `Decode` count as nothing, since Redis answered and the request was bad. A probe that gets one still reopens, through the dropped-probe rule in `circuit-breaker`. A pipeline counts one failure if any op came back `Transport`, else one success
|
||||
- An open breaker returns `Error::BreakerOpen` without touching the socket
|
||||
- Python doesn't count MASTERDOWN, TRYAGAIN or READONLY as health failures. Here they are `Transport` and do count
|
||||
|
||||
## Namespace
|
||||
- Applied once, here, to every key. A key that already starts with `{namespace}:` is left alone, matching `RedisCache.check_and_fix_namespace`
|
||||
- The rule never changes, because Python and Rust pods share keys in the same Redis
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-circuit-breaker.workspace = true
|
||||
futures-util.workspace = true
|
||||
redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] }
|
||||
thiserror.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
use std::time::Duration;
|
||||
use std::{
|
||||
future::Future,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use litellm_circuit_breaker::{Admission, Breaker, Failure};
|
||||
use redis::{Cmd, FromRedisValue, IntoConnectionInfo, ToRedisArgs, ToSingleRedisArg, Value};
|
||||
|
||||
use crate::{
|
||||
|
|
@ -11,6 +15,7 @@ use crate::{
|
|||
pub struct RedisClient {
|
||||
pub(crate) connection: Connection,
|
||||
namespace: Namespace,
|
||||
breaker: Option<Breaker>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
|
|
@ -34,6 +39,7 @@ impl RedisClient {
|
|||
Ok(Self {
|
||||
connection,
|
||||
namespace: config.namespace,
|
||||
breaker: config.breaker,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -110,10 +116,13 @@ impl RedisClient {
|
|||
) -> Result<T, Error> {
|
||||
let keys = self.keys(keys);
|
||||
let args = args.to_redis_args();
|
||||
Ok(script
|
||||
.invocation(&keys, &args)
|
||||
.invoke_async(&mut self.connection.clone())
|
||||
.await?)
|
||||
self.guarded(async {
|
||||
Ok(script
|
||||
.invocation(&keys, &args)
|
||||
.invoke_async(&mut self.connection.clone())
|
||||
.await?)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub fn pipeline(&self) -> Pipeline {
|
||||
|
|
@ -121,7 +130,23 @@ impl RedisClient {
|
|||
}
|
||||
|
||||
pub async fn run(&self, pipeline: &Pipeline) -> Vec<Result<Value, Error>> {
|
||||
pipeline.run(&self.connection).await
|
||||
let admission = match self.admit() {
|
||||
Ok(admission) => admission,
|
||||
Err(_) => {
|
||||
return (0..pipeline.len())
|
||||
.map(|_| Err(Error::BreakerOpen))
|
||||
.collect();
|
||||
}
|
||||
};
|
||||
let results = pipeline.run(&self.connection).await;
|
||||
let transport = results.iter().find_map(|result| {
|
||||
result
|
||||
.as_ref()
|
||||
.err()
|
||||
.filter(|error| matches!(error, Error::Transport(_)))
|
||||
});
|
||||
settle(admission, transport);
|
||||
results
|
||||
}
|
||||
|
||||
fn key(&self, key: &str) -> String {
|
||||
|
|
@ -133,7 +158,40 @@ impl RedisClient {
|
|||
}
|
||||
|
||||
async fn query<T: FromRedisValue>(&self, command: &Cmd) -> Result<T, Error> {
|
||||
Ok(command.query_async(&mut self.connection.clone()).await?)
|
||||
self.guarded(async { Ok(command.query_async(&mut self.connection.clone()).await?) })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn guarded<T>(&self, call: impl Future<Output = Result<T, Error>>) -> Result<T, Error> {
|
||||
let admission = self.admit()?;
|
||||
let result = call.await;
|
||||
settle(admission, result.as_ref().err());
|
||||
result
|
||||
}
|
||||
|
||||
fn admit(&self) -> Result<Option<Admission>, Error> {
|
||||
self.breaker
|
||||
.as_ref()
|
||||
.map(|breaker| {
|
||||
breaker
|
||||
.admit(Instant::now())
|
||||
.map_err(|_| Error::BreakerOpen)
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
}
|
||||
|
||||
fn settle(admission: Option<Admission>, error: Option<&Error>) {
|
||||
let Some(admission) = admission else {
|
||||
return;
|
||||
};
|
||||
match error {
|
||||
None => admission.succeeded(),
|
||||
Some(Error::Transport(error)) if error.is_timeout() => {
|
||||
admission.failed(Failure::Timeout, Instant::now())
|
||||
}
|
||||
Some(Error::Transport(_)) => admission.failed(Failure::Hard, Instant::now()),
|
||||
Some(Error::Server(_) | Error::Decode(_) | Error::BreakerOpen) => drop(admission),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -144,6 +202,8 @@ mod tests {
|
|||
use litellm_redis_testing::RedisServer;
|
||||
use rstest::rstest;
|
||||
|
||||
use litellm_circuit_breaker::{Breaker, Config, State};
|
||||
|
||||
use super::{RedisClient, Ttl};
|
||||
use crate::{
|
||||
DEFAULT_TIMEOUT, Endpoint, Error, HostEndpoint, Namespace, RedisConfig,
|
||||
|
|
@ -399,6 +459,122 @@ mod tests {
|
|||
assert_eq!(client.get::<i64>("shared").await.unwrap(), Some(1000));
|
||||
}
|
||||
|
||||
fn breaker(failure_threshold: u32) -> Breaker {
|
||||
Breaker::new(
|
||||
Config {
|
||||
failure_threshold,
|
||||
recovery_timeout: Duration::from_millis(200),
|
||||
timeout_min_duration: Duration::ZERO,
|
||||
},
|
||||
Box::new(|_| {}),
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_open_breaker_rejects_without_waiting_on_redis_and_a_probe_closes_it(
|
||||
#[values(Topology::Standalone, Topology::Cluster)] topology: Topology,
|
||||
) {
|
||||
let server = Server::start(topology).await;
|
||||
let breaker = breaker(2);
|
||||
let client = server
|
||||
.guarded_client("", Duration::from_millis(300), Some(breaker.clone()))
|
||||
.await;
|
||||
|
||||
server.freeze().await;
|
||||
let first = client.get::<String>("k").await;
|
||||
let second = client.get::<String>("k").await;
|
||||
let started = Instant::now();
|
||||
let third = client.get::<String>("k").await;
|
||||
let rejected_in = started.elapsed();
|
||||
server.thaw().await;
|
||||
|
||||
assert!(matches!(first, Err(Error::Transport(_))), "{first:?}");
|
||||
assert!(matches!(second, Err(Error::Transport(_))), "{second:?}");
|
||||
assert!(matches!(third, Err(Error::BreakerOpen)), "{third:?}");
|
||||
assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}");
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let recovered = loop {
|
||||
match client.get::<String>("k").await {
|
||||
Ok(value) => break Some(value),
|
||||
Err(_) if Instant::now() < deadline => {
|
||||
tokio::time::sleep(Duration::from_millis(50)).await
|
||||
}
|
||||
Err(_) => break None,
|
||||
}
|
||||
};
|
||||
assert_eq!(recovered, Some(None));
|
||||
assert!(
|
||||
matches!(breaker.state(), State::Closed { .. }),
|
||||
"{:?}",
|
||||
breaker.state()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn rejected_commands_never_open_the_breaker(
|
||||
#[values(Topology::Standalone, Topology::Cluster)] topology: Topology,
|
||||
) {
|
||||
let server = Server::start(topology).await;
|
||||
let breaker = breaker(2);
|
||||
let client = server
|
||||
.guarded_client("", DEFAULT_TIMEOUT, Some(breaker.clone()))
|
||||
.await;
|
||||
client.rpush("list", "item").await.unwrap();
|
||||
|
||||
let rejected = [
|
||||
client.incr_by("list", 1).await,
|
||||
client.incr_by("list", 1).await,
|
||||
client.incr_by("list", 1).await,
|
||||
];
|
||||
|
||||
assert!(
|
||||
matches!(rejected[2], Err(Error::Server(_))),
|
||||
"{:?}",
|
||||
rejected[2]
|
||||
);
|
||||
assert!(
|
||||
matches!(breaker.state(), State::Closed { hard: 0, .. }),
|
||||
"{:?}",
|
||||
breaker.state()
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_open_breaker_fails_every_pipeline_op_without_sending(
|
||||
#[values(Topology::Standalone, Topology::Cluster)] topology: Topology,
|
||||
) {
|
||||
let server = Server::start(topology).await;
|
||||
let client = server
|
||||
.guarded_client("", Duration::from_millis(300), Some(breaker(1)))
|
||||
.await;
|
||||
let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1);
|
||||
|
||||
server.freeze().await;
|
||||
let failed = client.run(&pipeline).await;
|
||||
let rejected = client.run(&pipeline).await;
|
||||
server.thaw().await;
|
||||
|
||||
assert!(
|
||||
matches!(failed[0], Err(Error::Transport(_))),
|
||||
"{:?}",
|
||||
failed[0]
|
||||
);
|
||||
assert!(
|
||||
matches!(rejected[0], Err(Error::BreakerOpen)),
|
||||
"{:?}",
|
||||
rejected[0]
|
||||
);
|
||||
assert!(
|
||||
matches!(rejected[1], Err(Error::BreakerOpen)),
|
||||
"{:?}",
|
||||
rejected[1]
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_host_endpoint_connects_to_its_database() {
|
||||
|
|
@ -409,6 +585,7 @@ mod tests {
|
|||
endpoint,
|
||||
namespace: Namespace::default(),
|
||||
timeout: DEFAULT_TIMEOUT,
|
||||
breaker: None,
|
||||
})
|
||||
};
|
||||
let in_db_3 = connect(Endpoint::Host(HostEndpoint {
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ use std::time::Duration;
|
|||
|
||||
use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult};
|
||||
|
||||
use litellm_circuit_breaker::Breaker;
|
||||
|
||||
use crate::Namespace;
|
||||
|
||||
pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
|
@ -10,6 +12,7 @@ pub struct RedisConfig {
|
|||
pub endpoint: Endpoint,
|
||||
pub namespace: Namespace,
|
||||
pub timeout: Duration,
|
||||
pub breaker: Option<Breaker>,
|
||||
}
|
||||
|
||||
pub enum Endpoint {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ pub enum Error {
|
|||
Server(#[source] RedisError),
|
||||
#[error("Redis replied with a value of a different type")]
|
||||
Decode(#[source] RedisError),
|
||||
#[error("the Redis circuit breaker is open, so the command was not sent")]
|
||||
BreakerOpen,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
|
|
|
|||
|
|
@ -93,6 +93,10 @@ impl Pipeline {
|
|||
self
|
||||
}
|
||||
|
||||
pub(crate) fn len(&self) -> usize {
|
||||
self.ops.len()
|
||||
}
|
||||
|
||||
pub(crate) async fn run(&self, connection: &Connection) -> Vec<Result<Value, Error>> {
|
||||
let groups = self.groups(connection);
|
||||
let settled = join_all(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::time::Duration;
|
||||
|
||||
use litellm_circuit_breaker::Breaker;
|
||||
use litellm_redis_testing::{RedisCluster, RedisServer};
|
||||
use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, SlotAddr};
|
||||
|
||||
|
|
@ -46,10 +47,20 @@ impl Server {
|
|||
}
|
||||
|
||||
pub(crate) async fn client(&self, namespace: &str, timeout: Duration) -> RedisClient {
|
||||
self.guarded_client(namespace, timeout, None).await
|
||||
}
|
||||
|
||||
pub(crate) async fn guarded_client(
|
||||
&self,
|
||||
namespace: &str,
|
||||
timeout: Duration,
|
||||
breaker: Option<Breaker>,
|
||||
) -> RedisClient {
|
||||
RedisClient::connect(RedisConfig {
|
||||
endpoint: self.endpoint(),
|
||||
namespace: Namespace::new(namespace),
|
||||
timeout,
|
||||
breaker,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue