feat(server): add circuit breaker pattern for external service calls

This commit is contained in:
Brad Groux 2026-01-29 05:01:15 -06:00
parent d36ee28a5d
commit 46eff13fa3
4 changed files with 355 additions and 1 deletions

View file

@ -2032,3 +2032,78 @@
{"type":"task.created","taskId":"task_20260129_ZNRHEG","status":"todo","id":"evt_kBBru2Zd2kkC","timestamp":"2026-01-29T11:00:12.510Z"}
{"type":"task.created","taskId":"task_20260129_jL6MHx","status":"todo","id":"evt_GNIFCXstbW46","timestamp":"2026-01-29T11:00:12.515Z"}
{"type":"task.status_changed","taskId":"task_20260129_OhJVTe","status":"done","previousStatus":"in-progress","id":"evt_6jBQKWsl7y9z","timestamp":"2026-01-29T11:00:22.085Z"}
{"type":"task.status_changed","taskId":"task_20260126_legacy1","status":"blocked","previousStatus":"review","id":"evt_CQqZJTtApL7P","timestamp":"2026-01-29T11:00:56.120Z"}
{"type":"task.created","taskId":"task_20260129_kVKZQH","status":"todo","id":"evt_ey0OJo99yktp","timestamp":"2026-01-29T11:00:56.154Z"}
{"type":"task.created","taskId":"task_20260129_PQKgJH","status":"todo","id":"evt_OTEatPZtWezB","timestamp":"2026-01-29T11:00:56.232Z"}
{"type":"task.status_changed","taskId":"task_20260129_PQKgJH","status":"in-progress","previousStatus":"todo","id":"evt_J3oTbFA9cC3V","timestamp":"2026-01-29T11:00:56.236Z"}
{"type":"task.created","taskId":"task_20260129_dn_MnL","status":"todo","id":"evt_iQtKb68u3Evq","timestamp":"2026-01-29T11:00:56.240Z"}
{"type":"task.status_changed","taskId":"task_20260129_dn_MnL","status":"blocked","previousStatus":"todo","id":"evt_GTLMjjr8PfDz","timestamp":"2026-01-29T11:00:56.242Z"}
{"type":"task.created","taskId":"task_20260129_U-fU9R","status":"todo","id":"evt_CkJF0bib9K6e","timestamp":"2026-01-29T11:00:56.244Z"}
{"type":"task.status_changed","taskId":"task_20260129_U-fU9R","status":"done","previousStatus":"todo","id":"evt_APBEHUbClnp2","timestamp":"2026-01-29T11:00:56.249Z"}
{"type":"task.created","taskId":"task_20260129_Bm9fj-","project":"my-project","status":"todo","id":"evt_aEkpBmKJMFHb","timestamp":"2026-01-29T11:00:56.254Z"}
{"type":"task.status_changed","taskId":"task_20260126_idem1","status":"blocked","previousStatus":"review","id":"evt_p5p_mXQfSWMj","timestamp":"2026-01-29T11:00:56.262Z"}
{"type":"task.created","taskId":"task_20260129_y7LoIv","status":"todo","id":"evt_A6GNL9iVWjgT","timestamp":"2026-01-29T11:00:56.266Z"}
{"type":"task.status_changed","taskId":"task_20260126_multi1","status":"blocked","previousStatus":"review","id":"evt_qyJN2mmC7aaC","timestamp":"2026-01-29T11:00:56.294Z"}
{"type":"task.status_changed","taskId":"task_20260126_multi2","status":"blocked","previousStatus":"review","id":"evt_XccRmveS4rbh","timestamp":"2026-01-29T11:00:56.297Z"}
{"type":"task.created","taskId":"task_20260129_QKi7Y7","status":"todo","id":"evt_HP3Fe9ha9Je6","timestamp":"2026-01-29T11:00:56.301Z"}
{"type":"task.created","taskId":"task_20260129_pr4stF","status":"todo","id":"evt_7So3GHnLsekk","timestamp":"2026-01-29T11:00:56.312Z"}
{"type":"task.created","taskId":"task_20260129_eAMcXG","status":"todo","id":"evt_N3jzGuIwUnyn","timestamp":"2026-01-29T11:00:56.326Z"}
{"type":"task.status_changed","taskId":"task_20260129_pr4stF","status":"in-progress","previousStatus":"todo","id":"evt_jpP52g3he8ma","timestamp":"2026-01-29T11:00:56.326Z"}
{"type":"task.created","taskId":"task_20260129_WfN5z-","status":"todo","id":"evt_AjVxN_Y-qvpD","timestamp":"2026-01-29T11:00:56.343Z"}
{"type":"task.created","taskId":"task_20260129_EWdbRj","status":"todo","id":"evt_gYCLMX24kJLJ","timestamp":"2026-01-29T11:00:56.353Z"}
{"type":"task.created","taskId":"task_20260129_q9N2qm","status":"todo","id":"evt_q8Dihc8R_eHr","timestamp":"2026-01-29T11:00:56.367Z"}
{"type":"task.created","taskId":"task_20260129_yD_HXl","status":"todo","id":"evt_ztzR9Cn-w2r6","timestamp":"2026-01-29T11:00:56.401Z"}
{"type":"task.restored","taskId":"task_20260126_arch1","status":"done","id":"evt__i4TrXdBBfPQ","timestamp":"2026-01-29T11:00:56.402Z"}
{"type":"task.status_changed","taskId":"task_20260126_arch1","status":"blocked","previousStatus":"done","id":"evt_e33eL0oftCf1","timestamp":"2026-01-29T11:00:56.404Z"}
{"type":"task.archived","taskId":"task_20260126_arch1","status":"blocked","id":"evt_oU0YsfSu7Lhf","timestamp":"2026-01-29T11:00:56.407Z"}
{"type":"task.created","taskId":"task_20260129_R162ak","status":"todo","id":"evt_lkjqAZaPOa1N","timestamp":"2026-01-29T11:00:56.412Z"}
{"type":"task.archived","taskId":"task_20260129_R162ak","status":"todo","id":"evt_zsSFYMoLnTt9","timestamp":"2026-01-29T11:00:56.417Z"}
{"type":"task.created","taskId":"task_20260129_bNsxNA","status":"todo","id":"evt_m1heK6118WnJ","timestamp":"2026-01-29T11:00:56.418Z"}
{"type":"task.created","taskId":"task_20260129_hAAOmh","status":"todo","id":"evt_9MsRYOV1q0s2","timestamp":"2026-01-29T11:00:56.432Z"}
{"type":"task.created","taskId":"task_20260129_OPqfGP","status":"todo","id":"evt_dIlJAf6ZB2qq","timestamp":"2026-01-29T11:00:56.433Z"}
{"type":"task.created","taskId":"task_20260129_H7aQA0","status":"todo","id":"evt_vUe7VXFP8uYl","timestamp":"2026-01-29T11:00:56.443Z"}
{"type":"task.created","taskId":"task_20260129_u5HdwO","status":"todo","id":"evt_tY1K65I3WO67","timestamp":"2026-01-29T11:00:56.443Z"}
{"type":"task.created","taskId":"task_20260129_aGas7T","status":"todo","id":"evt_XHZ7gDgYy7XF","timestamp":"2026-01-29T11:00:56.448Z"}
{"type":"task.created","taskId":"task_20260129_bJuCy3","status":"todo","id":"evt_vmlGnZhbP2Gj","timestamp":"2026-01-29T11:00:56.461Z"}
{"type":"task.created","taskId":"task_20260129_6L14q9","status":"todo","id":"evt_FpSwRCK-iDC8","timestamp":"2026-01-29T11:00:56.465Z"}
{"type":"task.created","taskId":"task_20260129_JvpxwV","status":"todo","id":"evt_WxTd1dPjXhyI","timestamp":"2026-01-29T11:00:56.472Z"}
{"type":"task.created","taskId":"task_20260129_Gyda-V","status":"todo","id":"evt_1JTRViR6rgak","timestamp":"2026-01-29T11:00:56.483Z"}
{"type":"task.created","taskId":"task_20260129_yE-d91","status":"todo","id":"evt_kMR87j8VKAdv","timestamp":"2026-01-29T11:00:56.495Z"}
{"type":"task.created","taskId":"task_20260129_7g4JMJ","status":"todo","id":"evt_p-X2wuQaAYBN","timestamp":"2026-01-29T11:00:56.510Z"}
{"type":"task.created","taskId":"task_20260129_94CfGH","status":"todo","id":"evt_y-Q9Vr0Au5vB","timestamp":"2026-01-29T11:00:56.513Z"}
{"type":"task.created","taskId":"task_20260129_QkeqLb","status":"todo","id":"evt_FW5rfCJhLT6j","timestamp":"2026-01-29T11:00:56.514Z"}
{"type":"task.created","taskId":"task_20260129_lWzYdy","status":"todo","id":"evt_CB0EhNXU5XsR","timestamp":"2026-01-29T11:00:56.520Z"}
{"type":"task.created","taskId":"task_20260129_ZtifT5","status":"todo","id":"evt_q1PcasdshEJW","timestamp":"2026-01-29T11:00:56.523Z"}
{"type":"task.created","taskId":"task_20260129_4A5NEH","status":"todo","id":"evt_DNBa-xrgQC2N","timestamp":"2026-01-29T11:00:56.531Z"}
{"type":"task.created","taskId":"task_20260129_s-2_jQ","status":"todo","id":"evt_ZDX9Ckue_-Vu","timestamp":"2026-01-29T11:00:56.534Z"}
{"type":"task.created","taskId":"task_20260129_g6qIeS","status":"todo","id":"evt_bkT6Eu_4h3LT","timestamp":"2026-01-29T11:00:56.540Z"}
{"type":"task.created","taskId":"task_20260129_YEETDq","status":"todo","id":"evt_sQPzgfs6upI5","timestamp":"2026-01-29T11:00:56.549Z"}
{"type":"task.created","taskId":"task_20260129_NrzxJ8","status":"todo","id":"evt_GCGzWG8LNFnS","timestamp":"2026-01-29T11:00:56.568Z"}
{"type":"task.created","taskId":"task_20260129_1oDC5J","project":"project-a","status":"todo","id":"evt_cigEwGRW3BUk","timestamp":"2026-01-29T11:00:56.591Z"}
{"type":"task.created","taskId":"task_20260129_AevP9H","project":"project-a","status":"todo","id":"evt_zNlfaSjvuo0i","timestamp":"2026-01-29T11:00:56.598Z"}
{"type":"task.created","taskId":"task_20260129_RUlJt5","project":"project-b","status":"todo","id":"evt_71JOaR6PWgbX","timestamp":"2026-01-29T11:00:56.600Z"}
{"type":"task.created","taskId":"task_20260129_3uEvat","status":"todo","id":"evt_BN-Fo4OpIcHX","timestamp":"2026-01-29T11:00:56.616Z"}
{"type":"task.created","taskId":"task_20260129_xKAZY_","status":"todo","id":"evt_jrF4fSpbqBOi","timestamp":"2026-01-29T11:00:56.676Z"}
{"type":"task.created","taskId":"task_20260129_u4Dzc5","status":"todo","id":"evt_VWmkHL1WV62K","timestamp":"2026-01-29T11:00:56.711Z"}
{"type":"task.created","taskId":"task_20260129_Ib9_6x","status":"todo","id":"evt_Yzj2LO5ZR-Dg","timestamp":"2026-01-29T11:00:56.750Z"}
{"type":"task.created","taskId":"task_20260129_yG93b_","status":"todo","id":"evt_zXk8B7ZFUp6M","timestamp":"2026-01-29T11:00:56.766Z"}
{"type":"task.created","taskId":"task_20260129_c9pqK6","status":"todo","id":"evt_rG6RE51lO0mL","timestamp":"2026-01-29T11:00:56.774Z"}
{"type":"task.created","taskId":"task_20260129_uHAywG","status":"todo","id":"evt_tgXk3Mg92_N6","timestamp":"2026-01-29T11:00:56.775Z"}
{"type":"task.created","taskId":"task_20260129_dhISci","status":"todo","id":"evt_B7z8ObCO4KSC","timestamp":"2026-01-29T11:00:56.777Z"}
{"type":"task.created","taskId":"task_20260129_1bbok4","status":"todo","id":"evt_typW7orZgrYd","timestamp":"2026-01-29T11:00:56.787Z"}
{"type":"task.created","taskId":"task_20260129_1KZUqq","status":"todo","id":"evt_MuweL6lJGKEU","timestamp":"2026-01-29T11:00:56.793Z"}
{"type":"task.created","taskId":"task_20260129_q8sk48","project":"test-project","status":"todo","id":"evt_wPOPdWRgS1Es","timestamp":"2026-01-29T11:00:56.802Z"}
{"type":"task.created","taskId":"task_20260129_HHGl32","status":"todo","id":"evt_o-MJFiMpiCor","timestamp":"2026-01-29T11:00:56.808Z"}
{"type":"task.created","taskId":"task_20260129_Vr9NzL","status":"todo","id":"evt_mKX8ouXdJ2yj","timestamp":"2026-01-29T11:00:56.810Z"}
{"type":"task.created","taskId":"task_20260129_w-4r4c","status":"todo","id":"evt_TeDTjQZOlbO3","timestamp":"2026-01-29T11:00:56.818Z"}
{"type":"task.created","taskId":"task_20260129_rILlN4","status":"todo","id":"evt_jftUzvrGjmIu","timestamp":"2026-01-29T11:00:56.821Z"}
{"type":"task.status_changed","taskId":"task_20260129_rILlN4","status":"in-progress","previousStatus":"todo","id":"evt_9igpivD0xmsi","timestamp":"2026-01-29T11:00:56.827Z"}
{"type":"task.created","taskId":"task_20260129_RQNnh2","status":"todo","id":"evt_aC3ywX8x5XTN","timestamp":"2026-01-29T11:00:56.828Z"}
{"type":"task.created","taskId":"task_20260129_xDg0VP","status":"todo","id":"evt_As9Yr6bqHAWy","timestamp":"2026-01-29T11:00:56.840Z"}
{"type":"task.created","taskId":"task_20260129_sIOoRm","status":"todo","id":"evt_EraFeDTGhavW","timestamp":"2026-01-29T11:00:56.840Z"}
{"type":"task.created","taskId":"task_20260129_bkFS8v","status":"todo","id":"evt_IROh2QX7OAUA","timestamp":"2026-01-29T11:00:56.864Z"}
{"type":"task.created","taskId":"task_20260129_19yQWD","status":"todo","id":"evt_0IpaAlw_FgT5","timestamp":"2026-01-29T11:00:56.868Z"}
{"type":"task.created","taskId":"task_20260129_Zsxd7t","status":"todo","id":"evt_dFVpxVtc3SCp","timestamp":"2026-01-29T11:00:56.946Z"}
{"type":"task.created","taskId":"task_20260129_ocqa1q","status":"todo","id":"evt_UtAfutoL9Czz","timestamp":"2026-01-29T11:00:56.948Z"}
{"type":"task.created","taskId":"task_20260129_ak_EdA","status":"todo","id":"evt_yPmLdjWS9sHP","timestamp":"2026-01-29T11:00:56.952Z"}
{"type":"task.status_changed","taskId":"task_20260129_g7GLeb","status":"done","previousStatus":"in-progress","id":"evt_HHiPihFMG5uE","timestamp":"2026-01-29T11:01:11.941Z"}

View file

@ -1,4 +1,26 @@
[
{
"id": "activity_1769684471958_wz7v725l5",
"type": "comment_added",
"taskId": "task_20260129_g7GLeb",
"taskTitle": "[v1.1] STABILITY: Add circuit breaker pattern for external service calls",
"details": {
"author": "Veritas",
"preview": "Implemented circuit breaker pattern: CircuitBreake..."
},
"timestamp": "2026-01-29T11:01:11.958Z"
},
{
"id": "activity_1769684471941_y1ec68g6u",
"type": "status_changed",
"taskId": "task_20260129_g7GLeb",
"taskTitle": "[v1.1] STABILITY: Add circuit breaker pattern for external service calls",
"details": {
"from": "in-progress",
"status": "done"
},
"timestamp": "2026-01-29T11:01:11.941Z"
},
{
"id": "activity_1769684429997_a0gnv72cw",
"type": "comment_added",

View file

@ -1,5 +1,5 @@
{
"status": "idle",
"subAgentCount": 0,
"lastUpdated": "2026-01-29T11:00:12.397Z"
"lastUpdated": "2026-01-29T11:00:57.529Z"
}

View file

@ -0,0 +1,257 @@
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { CircuitBreaker, CircuitOpenError } from '../../services/circuit-breaker.js';
// Silence the pino logger during tests
vi.mock('../../lib/logger.js', () => ({
createLogger: () => ({
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
}),
}));
describe('CircuitBreaker', () => {
let breaker: CircuitBreaker;
beforeEach(() => {
vi.useFakeTimers();
breaker = new CircuitBreaker({
name: 'test',
failureThreshold: 3,
resetTimeout: 10_000,
monitorWindow: 30_000,
});
});
afterEach(() => {
vi.useRealTimers();
});
// -----------------------------------------------
// Basic behavior
// -----------------------------------------------
it('starts in closed state', () => {
expect(breaker.state).toBe('closed');
});
it('passes through successful calls in closed state', async () => {
const result = await breaker.execute(() => Promise.resolve('ok'));
expect(result).toBe('ok');
expect(breaker.state).toBe('closed');
});
it('passes through failures in closed state (below threshold)', async () => {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow('fail');
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow('fail');
// 2 failures, threshold is 3 → still closed
expect(breaker.state).toBe('closed');
});
// -----------------------------------------------
// Closed → Open transition
// -----------------------------------------------
it('transitions to open after reaching failure threshold', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow(
'fail'
);
}
expect(breaker.state).toBe('open');
});
// -----------------------------------------------
// Open state behavior
// -----------------------------------------------
it('rejects immediately when open', async () => {
// Trip the circuit
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
expect(breaker.state).toBe('open');
// Subsequent calls should be rejected without executing the function
const fn = vi.fn(() => Promise.resolve('should not run'));
await expect(breaker.execute(fn)).rejects.toThrow(CircuitOpenError);
expect(fn).not.toHaveBeenCalled();
});
it('CircuitOpenError has correct properties', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
try {
await breaker.execute(() => Promise.resolve('nope'));
expect.fail('Should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(CircuitOpenError);
const coe = err as CircuitOpenError;
expect(coe.circuitName).toBe('test');
expect(coe.nextAttempt).toBeInstanceOf(Date);
}
});
// -----------------------------------------------
// Open → Half-open transition
// -----------------------------------------------
it('transitions to half-open after reset timeout', async () => {
// Trip the circuit
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
expect(breaker.state).toBe('open');
// Advance past the reset timeout
vi.advanceTimersByTime(10_001);
expect(breaker.state).toBe('half-open');
});
// -----------------------------------------------
// Half-open: allows one request
// -----------------------------------------------
it('allows one request through in half-open state', async () => {
// Trip and wait for half-open
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
vi.advanceTimersByTime(10_001);
expect(breaker.state).toBe('half-open');
const result = await breaker.execute(() => Promise.resolve('recovered'));
expect(result).toBe('recovered');
});
// -----------------------------------------------
// Half-open → Closed (success)
// -----------------------------------------------
it('closes the circuit when half-open request succeeds', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
vi.advanceTimersByTime(10_001);
expect(breaker.state).toBe('half-open');
await breaker.execute(() => Promise.resolve('ok'));
expect(breaker.state).toBe('closed');
// Should now allow normal traffic again
const result = await breaker.execute(() => Promise.resolve('normal'));
expect(result).toBe('normal');
});
// -----------------------------------------------
// Half-open → Open (failure)
// -----------------------------------------------
it('reopens the circuit when half-open request fails', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
vi.advanceTimersByTime(10_001);
expect(breaker.state).toBe('half-open');
await expect(breaker.execute(() => Promise.reject(new Error('still failing')))).rejects.toThrow(
'still failing'
);
expect(breaker.state).toBe('open');
});
// -----------------------------------------------
// Sliding window — old failures expire
// -----------------------------------------------
it('resets old failures outside the monitor window', async () => {
// 2 failures
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
expect(breaker.state).toBe('closed');
// Advance past the monitor window so those 2 failures expire
vi.advanceTimersByTime(31_000);
// One more failure — total recent = 1, not 3
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
expect(breaker.state).toBe('closed');
});
// -----------------------------------------------
// Concurrent half-open — only one passes
// -----------------------------------------------
it('only allows one concurrent request in half-open state', async () => {
// Trip the circuit
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
vi.advanceTimersByTime(10_001);
expect(breaker.state).toBe('half-open');
// Create a slow promise that we control
let resolveFirst!: (value: string) => void;
const slowPromise = new Promise<string>((resolve) => {
resolveFirst = resolve;
});
// Start first request (this one should get through)
const firstCall = breaker.execute(() => slowPromise);
// Second concurrent request should be rejected
await expect(breaker.execute(() => Promise.resolve('second'))).rejects.toThrow(
CircuitOpenError
);
// Resolve the first request
resolveFirst('first');
const result = await firstCall;
expect(result).toBe('first');
expect(breaker.state).toBe('closed');
});
// -----------------------------------------------
// Status reporting
// -----------------------------------------------
it('reports correct status in closed state', () => {
const status = breaker.getStatus();
expect(status.state).toBe('closed');
expect(status.failures).toBe(0);
expect(status.lastFailure).toBeNull();
expect(status.nextAttempt).toBeNull();
});
it('reports correct status in open state', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
const status = breaker.getStatus();
expect(status.state).toBe('open');
expect(status.failures).toBe(3);
expect(status.lastFailure).not.toBeNull();
expect(status.nextAttempt).not.toBeNull();
});
// -----------------------------------------------
// Manual reset
// -----------------------------------------------
it('can be manually reset to closed', async () => {
for (let i = 0; i < 3; i++) {
await expect(breaker.execute(() => Promise.reject(new Error('fail')))).rejects.toThrow();
}
expect(breaker.state).toBe('open');
breaker.reset();
expect(breaker.state).toBe('closed');
const result = await breaker.execute(() => Promise.resolve('after reset'));
expect(result).toBe('after reset');
});
});