diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 003cb2997d2..71e8e2f5a2e 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3655,6 +3655,14 @@ dependencies = [ "strum", ] +[[package]] +name = "litellm-circuit-breaker" +version = "0.1.0" +dependencies = [ + "proptest", + "rstest", +] + [[package]] name = "litellm-config" version = "0.1.0" @@ -4293,6 +4301,7 @@ name = "litellm-redis-client" version = "0.1.0" dependencies = [ "futures-util", + "litellm-circuit-breaker", "litellm-redis-testing", "redis", "rstest", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 53ae6f92986..734c1af8324 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -69,6 +69,7 @@ litellm-token-counter-huggingface = { path = "crates/token-counter-huggingface" litellm-token-counter-tiktoken = { path = "crates/token-counter-tiktoken" } litellm-host-python = { path = "crates/host-python" } litellm-python-compat = { path = "crates/python-compat" } +litellm-circuit-breaker = { path = "crates/circuit-breaker" } litellm-redis-client = { path = "crates/redis-client" } litellm-redis-testing = { path = "crates/redis-testing" } diff --git a/litellm-rust/crates/circuit-breaker/AGENTS.md b/litellm-rust/crates/circuit-breaker/AGENTS.md new file mode 100644 index 00000000000..52ef0ca7435 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/AGENTS.md @@ -0,0 +1,14 @@ +- `fsm` is a generic state machine runtime and knows nothing about breakers. `breaker` is a generic circuit breaker and knows nothing about Redis. Deciding which errors count as failures belongs to the consumer +- `Machine::step` stays pure. `&self` is config and time comes in on the event, never read inside the machine. The only clock read is `Admission`'s drop, at the call-site boundary +- Transition rules are one `match (state, event)` per machine, with no macro DSL and no `_` arm that would hide a new state from the compiler +- `Shared` counts a transition, bumps the generation and calls the observer only when the state's enum variant changes, matching Python bumping `generation` only when `_state` changes. Data inside a variant (failure counters, `since`) changes without a transition +- The observer runs under the lock so transitions are seen in generation order. Keep it to logging or a metric +- A disabled breaker is `Option::None` at the consumer, not a flag inside the machine. `Config` comes in already resolved, reading `REDIS_CIRCUIT_BREAKER_*` env is the consumer's job + +## Parity with Python `RedisCircuitBreaker` (`litellm/caching/redis_cache.py`) +- The rstest table `matches_the_python_redis_circuit_breaker` was checked case by case against the Python breaker. A behavior change here needs the same check, by driving the Python class with the same steps +- Divergences, each on purpose + - monotonic `Instant` instead of `time.time()`, so a wall clock jump can't open or close the breaker + - a probe `Admission` dropped without an outcome records a hard failure and reopens. Python's probe that dies on `CancelledError` (a `BaseException`) records nothing and leaves the breaker HALF_OPEN for good, and a dropped Rust future is common + - `_failure_count` is dropped, it only fed a log line + - no `enabled` flag, see above diff --git a/litellm-rust/crates/circuit-breaker/Cargo.toml b/litellm-rust/crates/circuit-breaker/Cargo.toml new file mode 100644 index 00000000000..8eca448a2ac --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "litellm-circuit-breaker" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] + +[dev-dependencies] +proptest.workspace = true +rstest.workspace = true diff --git a/litellm-rust/crates/circuit-breaker/src/breaker.rs b/litellm-rust/crates/circuit-breaker/src/breaker.rs new file mode 100644 index 00000000000..d7ddaab4e49 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/breaker.rs @@ -0,0 +1,414 @@ +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::{Machine, Observer, Shared}; + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Config { + pub failure_threshold: u32, + pub recovery_timeout: Duration, + pub timeout_min_duration: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + failure_threshold: 5, + recovery_timeout: Duration::from_secs(60), + timeout_min_duration: Duration::from_secs(5), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Failure { + Hard, + Timeout, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum State { + Closed { + hard: u32, + timeouts: u32, + streak_started: Option, + }, + Open { + since: Instant, + }, + HalfOpen, +} + +impl State { + const CLOSED: Self = Self::Closed { + hard: 0, + timeouts: 0, + streak_started: None, + }; +} + +#[derive(Clone, Copy, Debug)] +pub enum Event { + Admit(Instant), + Succeeded, + Failed(Failure, Instant), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Output { + Admitted, + Probe, + Rejected, + Recorded, +} + +impl Machine for Config { + type State = State; + type Event = Event; + type Output = Output; + + fn step(&self, state: &State, event: Event) -> (State, Output) { + match (state, event) { + (State::Closed { .. }, Event::Admit(_)) => (state.clone(), Output::Admitted), + (State::Open { since }, Event::Admit(now)) + if now.duration_since(*since) > self.recovery_timeout => + { + (State::HalfOpen, Output::Probe) + } + (State::Open { .. } | State::HalfOpen, Event::Admit(_)) => { + (state.clone(), Output::Rejected) + } + (State::Closed { .. } | State::HalfOpen, Event::Succeeded) => { + (State::CLOSED, Output::Recorded) + } + (State::Open { .. }, Event::Succeeded) => (state.clone(), Output::Recorded), + (State::Closed { hard, .. }, Event::Failed(Failure::Hard, now)) => { + let hard = hard + 1; + let next = match hard >= self.failure_threshold { + true => State::Open { since: now }, + false => State::Closed { + hard, + timeouts: 0, + streak_started: None, + }, + }; + (next, Output::Recorded) + } + ( + State::Closed { + hard, + timeouts, + streak_started, + }, + Event::Failed(Failure::Timeout, now), + ) => { + let timeouts = timeouts + 1; + let started = streak_started.unwrap_or(now); + let opens = timeouts >= self.failure_threshold + && now.duration_since(started) >= self.timeout_min_duration; + let next = match opens { + true => State::Open { since: now }, + false => State::Closed { + hard: *hard, + timeouts, + streak_started: Some(started), + }, + }; + (next, Output::Recorded) + } + (State::Open { .. } | State::HalfOpen, Event::Failed(_, now)) => { + (State::Open { since: now }, Output::Recorded) + } + } + } +} + +#[derive(Clone)] +pub struct Breaker(Arc>); + +#[derive(Debug, PartialEq, Eq)] +pub struct Rejected; + +pub struct Admission { + breaker: Breaker, + generation: u64, + probe: bool, + settled: bool, +} + +impl Breaker { + pub fn new(config: Config, observer: Observer) -> Self { + Self(Arc::new(Shared::new(config, State::CLOSED, observer))) + } + + pub fn admit(&self, now: Instant) -> Result { + let (output, generation) = self.0.fire(Event::Admit(now)); + let probe = match output { + Output::Admitted => false, + Output::Probe => true, + Output::Rejected | Output::Recorded => return Err(Rejected), + }; + Ok(Admission { + breaker: self.clone(), + generation, + probe, + settled: false, + }) + } + + pub fn failed(&self, failure: Failure, now: Instant) { + self.0.fire(Event::Failed(failure, now)); + } + + pub fn state(&self) -> State { + self.0.state() + } +} + +impl Admission { + pub fn probe(&self) -> bool { + self.probe + } + + pub fn succeeded(mut self) { + self.settled = true; + self.breaker.0.fire_if(self.generation, Event::Succeeded); + } + + pub fn failed(mut self, failure: Failure, now: Instant) { + self.settled = true; + self.breaker.failed(failure, now); + } +} + +impl Drop for Admission { + fn drop(&mut self) { + if self.probe && !self.settled { + self.breaker.failed(Failure::Hard, Instant::now()); + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, + }; + + use proptest::prelude::*; + use rstest::rstest; + + use super::{Admission, Breaker, Config, Failure, State}; + use crate::Transition; + + #[derive(Clone, Copy, Debug)] + enum Step { + Fail(f64, Failure), + Admit(f64, bool), + Succeed(usize), + } + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Phase { + Closed, + Open, + HalfOpen, + } + + use Phase::{Closed as C, HalfOpen as H, Open as O}; + + fn hard(at: f64) -> Step { + Step::Fail(at, Failure::Hard) + } + + fn timeout(at: f64) -> Step { + Step::Fail(at, Failure::Timeout) + } + + fn five_hard_at_zero() -> Vec { + vec![hard(0.0); 5] + } + + fn phase(state: &State) -> Phase { + match state { + State::Closed { .. } => Phase::Closed, + State::Open { .. } => Phase::Open, + State::HalfOpen => Phase::HalfOpen, + } + } + + fn quiet() -> Breaker { + Breaker::new(Config::default(), Box::new(|_| {})) + } + + #[rstest] + #[case::five_hard_failures_open(vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), hard(4.0)], vec![C, C, C, C, O])] + #[case::a_success_resets_the_hard_failure_count( + vec![hard(0.0), hard(1.0), hard(2.0), hard(3.0), Step::Admit(4.0, true), Step::Succeed(4), hard(5.0), hard(6.0), hard(7.0), hard(8.0)], + vec![C; 10] + )] + #[case::a_timeout_streak_opens_only_after_the_min_duration( + vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), timeout(4.9), timeout(5.0)], + vec![C, C, C, C, C, O] + )] + #[case::a_hard_failure_restarts_the_timeout_streak( + vec![timeout(0.0), timeout(1.0), timeout(2.0), timeout(3.0), hard(4.0), timeout(5.0), timeout(6.0), timeout(7.0), timeout(8.0), timeout(9.9), timeout(10.0)], + vec![C, C, C, C, C, C, C, C, C, C, O] + )] + #[case::the_probe_waits_until_strictly_past_the_recovery_timeout( + [five_hard_at_zero(), vec![Step::Admit(60.0, false), Step::Admit(60.5, true)]].concat(), + vec![C, C, C, C, O, O, H] + )] + #[case::a_probe_success_closes( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Succeed(5)]].concat(), + vec![C, C, C, C, O, H, C] + )] + #[case::a_probe_failure_reopens_with_a_fresh_timer( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), hard(62.0), Step::Admit(122.0, false), Step::Admit(122.5, true)]].concat(), + vec![C, C, C, C, O, H, O, O, H] + )] + #[case::only_one_probe_at_a_time( + [five_hard_at_zero(), vec![Step::Admit(61.0, true), Step::Admit(61.5, false)]].concat(), + vec![C, C, C, C, O, H, H] + )] + #[case::a_late_failure_while_open_pushes_back_the_probe( + [five_hard_at_zero(), vec![hard(30.0), Step::Admit(61.0, false), Step::Admit(90.5, true)]].concat(), + vec![C, C, C, C, O, O, O, H] + )] + #[case::a_success_from_before_the_breaker_moved_is_ignored( + vec![Step::Admit(0.0, true), hard(1.0), hard(1.0), hard(1.0), hard(1.0), hard(1.0), Step::Admit(62.0, true), Step::Succeed(0), Step::Succeed(6)], + vec![C, C, C, C, C, O, H, H, C] + )] + fn matches_the_python_redis_circuit_breaker( + #[case] steps: Vec, + #[case] expected: Vec, + ) { + let breaker = quiet(); + let t0 = Instant::now(); + let mut admissions: HashMap = HashMap::new(); + + let phases: Vec = steps + .iter() + .enumerate() + .map(|(index, step)| { + match *step { + Step::Fail(at, failure) => { + breaker.failed(failure, t0 + Duration::from_secs_f64(at)) + } + Step::Admit(at, admitted) => { + let admission = breaker.admit(t0 + Duration::from_secs_f64(at)); + assert_eq!(admission.is_ok(), admitted, "step {index}"); + admissions.extend(admission.ok().map(|admission| (index, admission))); + } + Step::Succeed(admitted_at) => { + admissions.remove(&admitted_at).unwrap().succeeded() + } + } + phase(&breaker.state()) + }) + .collect(); + + assert_eq!(phases, expected); + } + + #[rstest] + fn a_dropped_probe_reopens_instead_of_wedging_half_open() { + let breaker = quiet(); + let t0 = Instant::now(); + (0..5).for_each(|_| breaker.failed(Failure::Hard, t0)); + let probe = breaker.admit(t0 + Duration::from_secs(61)).unwrap(); + assert!(probe.probe()); + + drop(probe); + + assert!(matches!(breaker.state(), State::Open { .. })); + } + + #[rstest] + fn a_dropped_regular_call_records_nothing() { + let breaker = quiet(); + let t0 = Instant::now(); + breaker.failed(Failure::Hard, t0); + + drop(breaker.admit(t0).unwrap()); + + assert_eq!( + breaker.state(), + State::Closed { + hard: 1, + timeouts: 0, + streak_started: None + } + ); + } + + #[derive(Clone, Copy, Debug)] + enum Action { + Admit, + SucceedOldest, + FailOldest(Failure), + Fail(Failure), + } + + fn action() -> impl Strategy { + let failure = prop_oneof![Just(Failure::Hard), Just(Failure::Timeout)]; + let action = prop_oneof![ + Just(Action::Admit), + Just(Action::SucceedOldest), + failure.clone().prop_map(Action::FailOldest), + failure.prop_map(Action::Fail), + ]; + (0u64..30_000, action) + } + + proptest! { + #[test] + fn open_never_closes_directly_and_each_half_open_visit_has_one_probe( + actions in prop::collection::vec(action(), 1..200) + ) { + let transitions: Arc>>> = Arc::default(); + let seen = Arc::clone(&transitions); + let config = Config { + failure_threshold: 3, + recovery_timeout: Duration::from_secs(10), + timeout_min_duration: Duration::from_secs(2), + }; + let breaker = Breaker::new(config, Box::new(move |t| seen.lock().unwrap().push(t.clone()))); + let t0 = Instant::now(); + let mut now = t0; + let mut pending: Vec = Vec::new(); + let mut probes_per_visit: Vec = Vec::new(); + + for (advance_ms, action) in actions { + now += Duration::from_millis(advance_ms); + match action { + Action::Admit => { + if let Ok(admission) = breaker.admit(now) { + if admission.probe() { + probes_per_visit.push(1); + } + pending.push(admission); + } + } + Action::SucceedOldest if !pending.is_empty() => pending.remove(0).succeeded(), + Action::FailOldest(failure) if !pending.is_empty() => pending.remove(0).failed(failure, now), + Action::SucceedOldest | Action::FailOldest(_) => {} + Action::Fail(failure) => breaker.failed(failure, now), + } + } + + let transitions = transitions.lock().unwrap(); + let open_to_closed = transitions + .iter() + .filter(|t| matches!((&t.from, &t.to), (State::Open { .. }, State::Closed { .. }))) + .count(); + let half_open_visits = transitions.iter().filter(|t| t.to == State::HalfOpen).count(); + prop_assert_eq!(open_to_closed, 0); + prop_assert_eq!(probes_per_visit.len(), half_open_visits); + } + } +} diff --git a/litellm-rust/crates/circuit-breaker/src/fsm.rs b/litellm-rust/crates/circuit-breaker/src/fsm.rs new file mode 100644 index 00000000000..1beb0f890c8 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/fsm.rs @@ -0,0 +1,174 @@ +use std::{ + fmt::Debug, + mem, + sync::{Mutex, PoisonError}, +}; + +pub trait Machine { + type State: Clone + Debug; + type Event; + type Output; + + fn step(&self, state: &Self::State, event: Self::Event) -> (Self::State, Self::Output); +} + +#[derive(Clone, Debug)] +pub struct Transition { + pub from: S, + pub to: S, + pub generation: u64, +} + +pub type Observer = Box) + Send + Sync>; + +pub struct Shared { + machine: M, + current: Mutex<(M::State, u64)>, + observer: Observer, +} + +impl Shared { + pub fn new(machine: M, initial: M::State, observer: Observer) -> Self { + Self { + machine, + current: Mutex::new((initial, 0)), + observer, + } + } + + pub fn fire(&self, event: M::Event) -> (M::Output, u64) { + let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner); + self.apply(&mut current, event) + } + + pub fn fire_if(&self, generation: u64, event: M::Event) -> Option { + let mut current = self.current.lock().unwrap_or_else(PoisonError::into_inner); + (current.1 == generation).then(|| self.apply(&mut current, event).0) + } + + pub fn state(&self) -> M::State { + self.current + .lock() + .unwrap_or_else(PoisonError::into_inner) + .0 + .clone() + } + + fn apply(&self, current: &mut (M::State, u64), event: M::Event) -> (M::Output, u64) { + let (to, output) = self.machine.step(¤t.0, event); + if mem::discriminant(&to) != mem::discriminant(¤t.0) { + let from = mem::replace(&mut current.0, to.clone()); + current.1 += 1; + (self.observer)(&Transition { + from, + to, + generation: current.1, + }); + } else { + current.0 = to; + } + (output, current.1) + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use rstest::rstest; + + use super::{Machine, Shared, Transition}; + + struct Wrapping(u32); + + #[derive(Clone, Debug, PartialEq, Eq)] + enum Counter { + Low(u32), + High(u32), + } + + enum Tick { + Up, + Stay, + } + + impl Machine for Wrapping { + type State = Counter; + type Event = Tick; + type Output = (); + + fn step(&self, state: &Counter, event: Tick) -> (Counter, ()) { + let value = match (state, event) { + (Counter::Low(_) | Counter::High(_), Tick::Stay) => return (state.clone(), ()), + (Counter::Low(value) | Counter::High(value), Tick::Up) => (value + 1) % self.0, + }; + match value < self.0 / 2 { + true => (Counter::Low(value), ()), + false => (Counter::High(value), ()), + } + } + } + + type Seen = Arc>>>; + + fn recorded(wrap: u32) -> (Shared, Seen) { + let transitions: Seen = Arc::default(); + let seen = Arc::clone(&transitions); + let shared = Shared::new( + Wrapping(wrap), + Counter::Low(0), + Box::new(move |transition| seen.lock().unwrap().push(transition.clone())), + ); + (shared, transitions) + } + + #[rstest] + fn concurrent_events_are_all_applied_and_transitions_are_observed_in_generation_order() { + let (shared, transitions) = recorded(4); + + std::thread::scope(|scope| { + for _ in 0..64 { + scope.spawn(|| { + for _ in 0..100 { + shared.fire(Tick::Up); + } + }); + } + }); + + assert_eq!(shared.state(), Counter::Low(6400 % 4)); + let generations: Vec = transitions + .lock() + .unwrap() + .iter() + .map(|t| t.generation) + .collect(); + assert_eq!(generations, (1..=3200).collect::>()); + } + + #[rstest] + fn a_step_that_keeps_the_variant_neither_counts_nor_notifies() { + let (shared, transitions) = recorded(4); + + let (_, after_inner_change) = shared.fire(Tick::Up); + let (_, after_self_loop) = shared.fire(Tick::Stay); + + assert_eq!(shared.state(), Counter::Low(1)); + assert_eq!((after_inner_change, after_self_loop), (0, 0)); + assert!(transitions.lock().unwrap().is_empty()); + } + + #[rstest] + fn fire_if_skips_an_event_from_an_older_generation() { + let (shared, _) = recorded(4); + shared.fire(Tick::Up); + let (_, generation) = shared.fire(Tick::Up); + + let stale = shared.fire_if(generation - 1, Tick::Up); + let current = shared.fire_if(generation, Tick::Up); + + assert!(stale.is_none()); + assert!(current.is_some()); + assert_eq!(shared.state(), Counter::High(3)); + } +} diff --git a/litellm-rust/crates/circuit-breaker/src/lib.rs b/litellm-rust/crates/circuit-breaker/src/lib.rs new file mode 100644 index 00000000000..e76228122d1 --- /dev/null +++ b/litellm-rust/crates/circuit-breaker/src/lib.rs @@ -0,0 +1,5 @@ +mod breaker; +mod fsm; + +pub use breaker::{Admission, Breaker, Config, Event, Failure, Output, Rejected, State}; +pub use fsm::{Machine, Observer, Shared, Transition}; diff --git a/litellm-rust/crates/redis-client/AGENTS.md b/litellm-rust/crates/redis-client/AGENTS.md index e0f1a2da392..c73a4ed8c4d 100644 --- a/litellm-rust/crates/redis-client/AGENTS.md +++ b/litellm-rust/crates/redis-client/AGENTS.md @@ -12,6 +12,13 @@ - Never swallow an error or turn it into a miss. The caller picks the fallback - Reads return `Option`: a missing key is `None`, and a value that doesn't decode to `T` (`"15.0"` as `i64`) is `Decode` +## Circuit breaker +- The `litellm-circuit-breaker` `Breaker` is injected through `RedisConfig::breaker` and shared by every clone. `None` turns it off +- Every public call (each command, `eval`, one `run` of a pipeline) takes one admission through `guarded` / `admit` in `client.rs`. Never call the connection around it +- `Transport` counts as a failure (`Timeout` when `is_timeout()`, else `Hard`). `Server` and `Decode` count as nothing, since Redis answered and the request was bad. A probe that gets one still reopens, through the dropped-probe rule in `circuit-breaker`. A pipeline counts one failure if any op came back `Transport`, else one success +- An open breaker returns `Error::BreakerOpen` without touching the socket +- Python doesn't count MASTERDOWN, TRYAGAIN or READONLY as health failures. Here they are `Transport` and do count + ## Namespace - Applied once, here, to every key. A key that already starts with `{namespace}:` is left alone, matching `RedisCache.check_and_fix_namespace` - The rule never changes, because Python and Rust pods share keys in the same Redis diff --git a/litellm-rust/crates/redis-client/Cargo.toml b/litellm-rust/crates/redis-client/Cargo.toml index e15bbb6e6a5..eab57b36423 100644 --- a/litellm-rust/crates/redis-client/Cargo.toml +++ b/litellm-rust/crates/redis-client/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-circuit-breaker.workspace = true futures-util.workspace = true redis = { workspace = true, features = ["cluster-async", "connection-manager", "tokio-rustls-comp"] } thiserror.workspace = true diff --git a/litellm-rust/crates/redis-client/src/client.rs b/litellm-rust/crates/redis-client/src/client.rs index 0de40a058a2..28fd16ecd23 100644 --- a/litellm-rust/crates/redis-client/src/client.rs +++ b/litellm-rust/crates/redis-client/src/client.rs @@ -1,5 +1,9 @@ -use std::time::Duration; +use std::{ + future::Future, + time::{Duration, Instant}, +}; +use litellm_circuit_breaker::{Admission, Breaker, Failure}; use redis::{Cmd, FromRedisValue, IntoConnectionInfo, ToRedisArgs, ToSingleRedisArg, Value}; use crate::{ @@ -11,6 +15,7 @@ use crate::{ pub struct RedisClient { pub(crate) connection: Connection, namespace: Namespace, + breaker: Option, } #[derive(Debug, PartialEq, Eq)] @@ -34,6 +39,7 @@ impl RedisClient { Ok(Self { connection, namespace: config.namespace, + breaker: config.breaker, }) } @@ -110,10 +116,13 @@ impl RedisClient { ) -> Result { let keys = self.keys(keys); let args = args.to_redis_args(); - Ok(script - .invocation(&keys, &args) - .invoke_async(&mut self.connection.clone()) - .await?) + self.guarded(async { + Ok(script + .invocation(&keys, &args) + .invoke_async(&mut self.connection.clone()) + .await?) + }) + .await } pub fn pipeline(&self) -> Pipeline { @@ -121,7 +130,23 @@ impl RedisClient { } pub async fn run(&self, pipeline: &Pipeline) -> Vec> { - pipeline.run(&self.connection).await + let admission = match self.admit() { + Ok(admission) => admission, + Err(_) => { + return (0..pipeline.len()) + .map(|_| Err(Error::BreakerOpen)) + .collect(); + } + }; + let results = pipeline.run(&self.connection).await; + let transport = results.iter().find_map(|result| { + result + .as_ref() + .err() + .filter(|error| matches!(error, Error::Transport(_))) + }); + settle(admission, transport); + results } fn key(&self, key: &str) -> String { @@ -133,7 +158,40 @@ impl RedisClient { } async fn query(&self, command: &Cmd) -> Result { - Ok(command.query_async(&mut self.connection.clone()).await?) + self.guarded(async { Ok(command.query_async(&mut self.connection.clone()).await?) }) + .await + } + + async fn guarded(&self, call: impl Future>) -> Result { + let admission = self.admit()?; + let result = call.await; + settle(admission, result.as_ref().err()); + result + } + + fn admit(&self) -> Result, Error> { + self.breaker + .as_ref() + .map(|breaker| { + breaker + .admit(Instant::now()) + .map_err(|_| Error::BreakerOpen) + }) + .transpose() + } +} + +fn settle(admission: Option, error: Option<&Error>) { + let Some(admission) = admission else { + return; + }; + match error { + None => admission.succeeded(), + Some(Error::Transport(error)) if error.is_timeout() => { + admission.failed(Failure::Timeout, Instant::now()) + } + Some(Error::Transport(_)) => admission.failed(Failure::Hard, Instant::now()), + Some(Error::Server(_) | Error::Decode(_) | Error::BreakerOpen) => drop(admission), } } @@ -144,6 +202,8 @@ mod tests { use litellm_redis_testing::RedisServer; use rstest::rstest; + use litellm_circuit_breaker::{Breaker, Config, State}; + use super::{RedisClient, Ttl}; use crate::{ DEFAULT_TIMEOUT, Endpoint, Error, HostEndpoint, Namespace, RedisConfig, @@ -399,6 +459,122 @@ mod tests { assert_eq!(client.get::("shared").await.unwrap(), Some(1000)); } + fn breaker(failure_threshold: u32) -> Breaker { + Breaker::new( + Config { + failure_threshold, + recovery_timeout: Duration::from_millis(200), + timeout_min_duration: Duration::ZERO, + }, + Box::new(|_| {}), + ) + } + + #[rstest] + #[tokio::test] + async fn an_open_breaker_rejects_without_waiting_on_redis_and_a_probe_closes_it( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let breaker = breaker(2); + let client = server + .guarded_client("", Duration::from_millis(300), Some(breaker.clone())) + .await; + + server.freeze().await; + let first = client.get::("k").await; + let second = client.get::("k").await; + let started = Instant::now(); + let third = client.get::("k").await; + let rejected_in = started.elapsed(); + server.thaw().await; + + assert!(matches!(first, Err(Error::Transport(_))), "{first:?}"); + assert!(matches!(second, Err(Error::Transport(_))), "{second:?}"); + assert!(matches!(third, Err(Error::BreakerOpen)), "{third:?}"); + assert!(rejected_in < Duration::from_millis(50), "{rejected_in:?}"); + + let deadline = Instant::now() + Duration::from_secs(30); + let recovered = loop { + match client.get::("k").await { + Ok(value) => break Some(value), + Err(_) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(50)).await + } + Err(_) => break None, + } + }; + assert_eq!(recovered, Some(None)); + assert!( + matches!(breaker.state(), State::Closed { .. }), + "{:?}", + breaker.state() + ); + } + + #[rstest] + #[tokio::test] + async fn rejected_commands_never_open_the_breaker( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let breaker = breaker(2); + let client = server + .guarded_client("", DEFAULT_TIMEOUT, Some(breaker.clone())) + .await; + client.rpush("list", "item").await.unwrap(); + + let rejected = [ + client.incr_by("list", 1).await, + client.incr_by("list", 1).await, + client.incr_by("list", 1).await, + ]; + + assert!( + matches!(rejected[2], Err(Error::Server(_))), + "{:?}", + rejected[2] + ); + assert!( + matches!(breaker.state(), State::Closed { hard: 0, .. }), + "{:?}", + breaker.state() + ); + } + + #[rstest] + #[tokio::test] + async fn an_open_breaker_fails_every_pipeline_op_without_sending( + #[values(Topology::Standalone, Topology::Cluster)] topology: Topology, + ) { + let server = Server::start(topology).await; + let client = server + .guarded_client("", Duration::from_millis(300), Some(breaker(1))) + .await; + let pipeline = client.pipeline().incr_by("a", 1).incr_by("c", 1); + + server.freeze().await; + let failed = client.run(&pipeline).await; + let rejected = client.run(&pipeline).await; + server.thaw().await; + + assert!( + matches!(failed[0], Err(Error::Transport(_))), + "{:?}", + failed[0] + ); + assert!( + matches!(rejected[0], Err(Error::BreakerOpen)), + "{:?}", + rejected[0] + ); + assert!( + matches!(rejected[1], Err(Error::BreakerOpen)), + "{:?}", + rejected[1] + ); + } + #[rstest] #[tokio::test] async fn a_host_endpoint_connects_to_its_database() { @@ -409,6 +585,7 @@ mod tests { endpoint, namespace: Namespace::default(), timeout: DEFAULT_TIMEOUT, + breaker: None, }) }; let in_db_3 = connect(Endpoint::Host(HostEndpoint { diff --git a/litellm-rust/crates/redis-client/src/config.rs b/litellm-rust/crates/redis-client/src/config.rs index d35666fc368..81dd532d0cf 100644 --- a/litellm-rust/crates/redis-client/src/config.rs +++ b/litellm-rust/crates/redis-client/src/config.rs @@ -2,6 +2,8 @@ use std::time::Duration; use redis::{ConnectionAddr, ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult}; +use litellm_circuit_breaker::Breaker; + use crate::Namespace; pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(5); @@ -10,6 +12,7 @@ pub struct RedisConfig { pub endpoint: Endpoint, pub namespace: Namespace, pub timeout: Duration, + pub breaker: Option, } pub enum Endpoint { diff --git a/litellm-rust/crates/redis-client/src/error.rs b/litellm-rust/crates/redis-client/src/error.rs index 61b9b2b89c0..ed3b2c74004 100644 --- a/litellm-rust/crates/redis-client/src/error.rs +++ b/litellm-rust/crates/redis-client/src/error.rs @@ -8,6 +8,8 @@ pub enum Error { Server(#[source] RedisError), #[error("Redis replied with a value of a different type")] Decode(#[source] RedisError), + #[error("the Redis circuit breaker is open, so the command was not sent")] + BreakerOpen, } #[derive(Debug, thiserror::Error)] diff --git a/litellm-rust/crates/redis-client/src/pipeline.rs b/litellm-rust/crates/redis-client/src/pipeline.rs index 74d659170e9..e54c65e9605 100644 --- a/litellm-rust/crates/redis-client/src/pipeline.rs +++ b/litellm-rust/crates/redis-client/src/pipeline.rs @@ -93,6 +93,10 @@ impl Pipeline { self } + pub(crate) fn len(&self) -> usize { + self.ops.len() + } + pub(crate) async fn run(&self, connection: &Connection) -> Vec> { let groups = self.groups(connection); let settled = join_all( diff --git a/litellm-rust/crates/redis-client/src/test_support.rs b/litellm-rust/crates/redis-client/src/test_support.rs index 34e78fc12d3..af1d1c46b9e 100644 --- a/litellm-rust/crates/redis-client/src/test_support.rs +++ b/litellm-rust/crates/redis-client/src/test_support.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use litellm_circuit_breaker::Breaker; use litellm_redis_testing::{RedisCluster, RedisServer}; use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, SlotAddr}; @@ -46,10 +47,20 @@ impl Server { } pub(crate) async fn client(&self, namespace: &str, timeout: Duration) -> RedisClient { + self.guarded_client(namespace, timeout, None).await + } + + pub(crate) async fn guarded_client( + &self, + namespace: &str, + timeout: Duration, + breaker: Option, + ) -> RedisClient { RedisClient::connect(RedisConfig { endpoint: self.endpoint(), namespace: Namespace::new(namespace), timeout, + breaker, }) .await .unwrap()