feat: define dependency circuit control semantics (#1097)

This commit is contained in:
Brad Groux 2026-07-25 23:39:10 -05:00 committed by GitHub
parent aa17223a06
commit 1e6869613b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 732 additions and 0 deletions

View file

@ -0,0 +1,256 @@
import { describe, expect, it } from 'vitest';
import type {
DependencyCircuitAdmission,
DependencyCircuitPolicy,
DependencyOutcome,
} from '@veritas-kanban/shared';
import {
DependencyCircuitBreaker,
classifyDependencyOutcome,
dependencyCircuitKey,
opaqueDependencyId,
} from '../services/dependency-circuit-breaker.js';
function policy(overrides: Partial<DependencyCircuitPolicy> = {}): DependencyCircuitPolicy {
return {
schemaVersion: 'dependency-circuit-policy/v1',
minimumSamples: 4,
rollingWindowMs: 10_000,
failureRateThreshold: 0.5,
slowCallDurationMs: 1_000,
slowCallRateThreshold: 0.5,
openDurationMs: 2_000,
openDurationJitterRatio: 0,
halfOpenMaxConcurrent: 1,
probeSuccessThreshold: 2,
...overrides,
};
}
function fixture(overrides: Partial<DependencyCircuitPolicy> = {}) {
let now = Date.parse('2026-07-25T12:00:00.000Z');
const breaker = new DependencyCircuitBreaker({
dependency: {
kind: 'model-endpoint',
id: 'openai-gpt',
workspaceId: 'workspace_1',
provider: 'codex-cli',
model: 'gpt-5.6',
},
policy: policy(overrides),
now: () => now,
jitter: () => 0.5,
});
return {
breaker,
advance: (milliseconds: number) => {
now += milliseconds;
},
};
}
function admitted(admission: DependencyCircuitAdmission) {
if (!admission.allowed) throw new Error('Expected the circuit to admit the call.');
return admission.lease;
}
function record(
breaker: DependencyCircuitBreaker,
outcome: DependencyOutcome,
durationMs = 10
) {
const lease = admitted(breaker.acquire());
breaker.record(lease, outcome, durationMs);
}
describe('DependencyCircuitBreaker', () => {
it('builds a stable scoped key while offering a hash for secret endpoint identities', () => {
const identity = {
kind: 'provider' as const,
id: opaqueDependencyId('https://secret-host.example/v1?token=do-not-store'),
workspaceId: 'workspace_1',
provider: 'codex-cli',
};
expect(identity.id).toMatch(/^dep_[A-Za-z0-9_-]{24}$/);
expect(identity.id).not.toContain('secret-host');
expect(dependencyCircuitKey(identity)).toBe(
`workspace_1:provider:${identity.id}:codex-cli:-:-`
);
});
it('requires the minimum sample size before evaluating failure rate', () => {
const { breaker } = fixture({ minimumSamples: 3, failureRateThreshold: 0.5 });
record(breaker, 'dependency-failure');
record(breaker, 'dependency-failure');
expect(breaker.getSnapshot().state).toBe('closed');
record(breaker, 'success');
expect(breaker.getSnapshot()).toMatchObject({
state: 'open',
reason: {
code: 'failure-rate',
sampleCount: 3,
failureCount: 2,
failureRate: 2 / 3,
},
});
});
it('opens exactly at the configured failure-rate boundary', () => {
const { breaker } = fixture();
record(breaker, 'success');
record(breaker, 'dependency-failure');
record(breaker, 'success');
record(breaker, 'timeout');
expect(breaker.getSnapshot()).toMatchObject({
state: 'open',
failureCount: 2,
failureRate: 0.5,
reason: { code: 'failure-rate' },
});
});
it('excludes caller cancellation, policy denial, and validation errors', () => {
const { breaker } = fixture({ minimumSamples: 2 });
record(breaker, 'caller-cancellation');
record(breaker, 'policy-block');
record(breaker, 'validation-error');
record(breaker, 'success');
expect(breaker.getSnapshot()).toMatchObject({
state: 'closed',
sampleCount: 1,
failureCount: 0,
lastOutcome: 'success',
});
});
it('opens on slow-call rate without treating slow success as dependency failure', () => {
const { breaker } = fixture();
record(breaker, 'success', 1_000);
record(breaker, 'success', 10);
record(breaker, 'success', 1_500);
record(breaker, 'success', 20);
expect(breaker.getSnapshot()).toMatchObject({
state: 'open',
failureCount: 0,
slowCallCount: 2,
slowCallRate: 0.5,
reason: { code: 'slow-call-rate' },
});
});
it('rejects while open and single-flights half-open probes', () => {
const { breaker, advance } = fixture({ minimumSamples: 1 });
record(breaker, 'dependency-failure');
expect(breaker.acquire()).toMatchObject({
allowed: false,
reason: 'circuit-open',
retryAt: '2026-07-25T12:00:02.000Z',
});
advance(2_000);
const probe = breaker.acquire();
expect(probe).toMatchObject({ allowed: true, decision: 'probe' });
expect(breaker.acquire()).toMatchObject({
allowed: false,
reason: 'probe-concurrency-exhausted',
});
});
it('closes only after the configured number of successful probes', () => {
const { breaker, advance } = fixture({ minimumSamples: 1 });
record(breaker, 'dependency-failure');
advance(2_000);
record(breaker, 'success');
expect(breaker.getSnapshot()).toMatchObject({ state: 'half-open', halfOpenSuccesses: 1 });
record(breaker, 'success');
expect(breaker.getSnapshot()).toMatchObject({
state: 'closed',
sampleCount: 0,
reason: { code: 'probe-succeeded' },
});
});
it('reopens after a failed probe and applies bounded jitter', () => {
const { breaker, advance } = fixture({
minimumSamples: 1,
openDurationJitterRatio: 0.25,
});
record(breaker, 'dependency-failure');
advance(2_000);
const probe = admitted(breaker.acquire());
breaker.record(probe, 'overload', 20);
expect(breaker.getSnapshot()).toMatchObject({
state: 'open',
reason: { code: 'probe-failed' },
nextProbeAt: '2026-07-25T12:00:04.000Z',
});
});
it('does not let a stale concurrent probe close a reopened circuit', () => {
const { breaker, advance } = fixture({
minimumSamples: 1,
halfOpenMaxConcurrent: 2,
probeSuccessThreshold: 1,
});
record(breaker, 'dependency-failure');
advance(2_000);
const failed = admitted(breaker.acquire());
const stale = admitted(breaker.acquire());
breaker.record(failed, 'timeout', 2_000);
breaker.record(stale, 'success', 10);
expect(breaker.getSnapshot()).toMatchObject({
state: 'open',
reason: { code: 'probe-failed' },
});
});
it('prunes old samples from the rolling window', () => {
const { breaker, advance } = fixture({ minimumSamples: 2, rollingWindowMs: 1_000 });
record(breaker, 'dependency-failure');
advance(1_001);
record(breaker, 'success');
expect(breaker.getSnapshot()).toMatchObject({
state: 'closed',
sampleCount: 1,
failureCount: 0,
});
});
it('rejects duplicate lease settlement', () => {
const { breaker } = fixture();
const lease = admitted(breaker.acquire());
breaker.record(lease, 'success', 10);
expect(() => breaker.record(lease, 'success', 10)).toThrow('missing, settled');
});
});
describe('classifyDependencyOutcome', () => {
it.each([
[{ succeeded: true }, 'success'],
[{ callerCancelled: true, timedOut: true }, 'caller-cancellation'],
[{ policyBlocked: true }, 'policy-block'],
[{ validationFailed: true }, 'validation-error'],
[{ timedOut: true }, 'timeout'],
[{ statusCode: 429 }, 'throttled'],
[{ statusCode: 503 }, 'overload'],
[{ errorCode: 'OVERLOADED' }, 'overload'],
[{}, 'dependency-failure'],
] as const)('classifies %j as %s', (signals, expected) => {
expect(classifyDependencyOutcome(signals)).toBe(expected);
});
});

View file

@ -0,0 +1,35 @@
import { z } from 'zod';
import {
DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
DEPENDENCY_KINDS,
type DependencyCircuitPolicy,
type DependencyIdentity,
} from '@veritas-kanban/shared';
const IdentifierSchema = z.string().trim().min(1).max(240);
export const DependencyIdentitySchema: z.ZodType<DependencyIdentity> = z
.object({
kind: z.enum(DEPENDENCY_KINDS),
id: IdentifierSchema.regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/),
workspaceId: IdentifierSchema.optional(),
provider: IdentifierSchema.optional(),
model: IdentifierSchema.optional(),
hostId: IdentifierSchema.optional(),
})
.strict();
export const DependencyCircuitPolicySchema: z.ZodType<DependencyCircuitPolicy> = z
.object({
schemaVersion: z.literal(DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION),
minimumSamples: z.number().int().min(1).max(100_000),
rollingWindowMs: z.number().int().min(1_000).max(24 * 60 * 60 * 1_000),
failureRateThreshold: z.number().min(0.01).max(1),
slowCallDurationMs: z.number().int().min(1).max(60 * 60 * 1_000),
slowCallRateThreshold: z.number().min(0.01).max(1),
openDurationMs: z.number().int().min(100).max(60 * 60 * 1_000),
openDurationJitterRatio: z.number().min(0).max(0.5),
halfOpenMaxConcurrent: z.number().int().min(1).max(1_000),
probeSuccessThreshold: z.number().int().min(1).max(10_000),
})
.strict();

View file

@ -0,0 +1,317 @@
import { createHash } from 'node:crypto';
import { nanoid } from 'nanoid';
import {
DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
DEPENDENCY_CIRCUIT_SCHEMA_VERSION,
type DependencyCircuitAdmission,
type DependencyCircuitLease,
type DependencyCircuitPolicy,
type DependencyCircuitReason,
type DependencyCircuitSnapshot,
type DependencyIdentity,
type DependencyOutcome,
type DependencyOutcomeSignals,
} from '@veritas-kanban/shared';
import {
DependencyCircuitPolicySchema,
DependencyIdentitySchema,
} from '../schemas/dependency-circuit-schemas.js';
export const DEFAULT_DEPENDENCY_CIRCUIT_POLICY: DependencyCircuitPolicy = {
schemaVersion: DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
minimumSamples: 10,
rollingWindowMs: 60_000,
failureRateThreshold: 0.5,
slowCallDurationMs: 30_000,
slowCallRateThreshold: 0.5,
openDurationMs: 30_000,
openDurationJitterRatio: 0.1,
halfOpenMaxConcurrent: 1,
probeSuccessThreshold: 2,
};
interface CircuitSample {
occurredAt: number;
outcome: DependencyOutcome;
durationMs: number;
}
interface ActiveLease {
lease: DependencyCircuitLease;
settled: boolean;
}
export interface DependencyCircuitBreakerOptions {
dependency: DependencyIdentity;
policy?: DependencyCircuitPolicy;
now?: () => number;
jitter?: () => number;
}
function isFailure(outcome: DependencyOutcome): boolean {
return ['dependency-failure', 'timeout', 'throttled', 'overload'].includes(outcome);
}
function isExcluded(outcome: DependencyOutcome): boolean {
return ['caller-cancellation', 'policy-block', 'validation-error'].includes(outcome);
}
export function dependencyCircuitKey(identity: DependencyIdentity): string {
const parsed = DependencyIdentitySchema.parse(identity);
return [
parsed.workspaceId ?? 'global',
parsed.kind,
parsed.id,
parsed.provider ?? '-',
parsed.model ?? '-',
parsed.hostId ?? '-',
].join(':');
}
export function opaqueDependencyId(value: string): string {
return `dep_${createHash('sha256').update(value).digest('base64url').slice(0, 24)}`;
}
export function classifyDependencyOutcome(signals: DependencyOutcomeSignals): DependencyOutcome {
if (signals.callerCancelled) return 'caller-cancellation';
if (signals.policyBlocked) return 'policy-block';
if (signals.validationFailed) return 'validation-error';
if (signals.timedOut || signals.errorCode === 'ETIMEDOUT') return 'timeout';
if (signals.statusCode === 429 || signals.errorCode === 'RATE_LIMITED') return 'throttled';
if (
signals.statusCode === 503 ||
signals.statusCode === 529 ||
signals.errorCode === 'OVERLOADED'
) {
return 'overload';
}
if (signals.succeeded) return 'success';
return 'dependency-failure';
}
export class DependencyCircuitBreaker {
readonly key: string;
readonly dependency: DependencyIdentity;
readonly policy: DependencyCircuitPolicy;
private readonly now: () => number;
private readonly jitter: () => number;
private stateValue: DependencyCircuitSnapshot['state'] = 'closed';
private samples: CircuitSample[] = [];
private leases = new Map<string, ActiveLease>();
private reason?: DependencyCircuitReason;
private openedAt?: number;
private nextProbeAt?: number;
private halfOpenSuccesses = 0;
private lastOutcome?: DependencyOutcome;
private lastOutcomeAt?: number;
constructor(options: DependencyCircuitBreakerOptions) {
this.dependency = DependencyIdentitySchema.parse(options.dependency);
this.policy = DependencyCircuitPolicySchema.parse(
options.policy ?? DEFAULT_DEPENDENCY_CIRCUIT_POLICY
);
this.key = dependencyCircuitKey(this.dependency);
this.now = options.now ?? Date.now;
this.jitter = options.jitter ?? Math.random;
}
acquire(): DependencyCircuitAdmission {
const now = this.now();
this.prune(now);
if (
this.stateValue === 'open' &&
this.nextProbeAt !== undefined &&
now >= this.nextProbeAt
) {
this.stateValue = 'half-open';
this.halfOpenSuccesses = 0;
this.reason = this.buildReason('probe-window-opened', now);
}
if (this.stateValue === 'open') {
return {
allowed: false,
decision: 'reject',
reason: 'circuit-open',
retryAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
snapshot: this.snapshot(now),
};
}
const probe = this.stateValue === 'half-open';
if (probe && this.activeProbeCount() >= this.policy.halfOpenMaxConcurrent) {
return {
allowed: false,
decision: 'reject',
reason: 'probe-concurrency-exhausted',
retryAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
snapshot: this.snapshot(now),
};
}
const lease: DependencyCircuitLease = {
id: `cirlease_${nanoid(18)}`,
circuitKey: this.key,
probe,
acquiredAt: new Date(now).toISOString(),
};
this.leases.set(lease.id, { lease, settled: false });
return {
allowed: true,
decision: probe ? 'probe' : 'allow',
lease,
snapshot: this.snapshot(now),
};
}
record(lease: DependencyCircuitLease, outcome: DependencyOutcome, durationMs: number): void {
const active = this.leases.get(lease.id);
if (!active || active.settled || active.lease.circuitKey !== this.key) {
throw new Error('Dependency circuit lease is missing, settled, or belongs to another circuit.');
}
if (!Number.isFinite(durationMs) || durationMs < 0) {
throw new Error('Dependency circuit duration must be a non-negative finite number.');
}
active.settled = true;
this.leases.delete(lease.id);
const now = this.now();
this.lastOutcome = outcome;
this.lastOutcomeAt = now;
if (!isExcluded(outcome)) {
const normalizedOutcome =
outcome === 'success' && durationMs >= this.policy.slowCallDurationMs
? 'slow-success'
: outcome;
this.samples.push({ occurredAt: now, outcome: normalizedOutcome, durationMs });
this.lastOutcome = normalizedOutcome;
}
this.prune(now);
if (!lease.probe) {
this.evaluateClosed(now);
return;
}
if (this.stateValue !== 'half-open') return;
if (isExcluded(outcome)) return;
const failed =
isFailure(this.lastOutcome ?? outcome) ||
durationMs >= this.policy.slowCallDurationMs;
if (failed) {
this.open('probe-failed', now);
return;
}
this.halfOpenSuccesses += 1;
if (this.halfOpenSuccesses >= this.policy.probeSuccessThreshold) {
this.close('probe-succeeded', now);
}
}
reset(): void {
const now = this.now();
this.samples = [];
this.leases.clear();
this.stateValue = 'closed';
this.openedAt = undefined;
this.nextProbeAt = undefined;
this.halfOpenSuccesses = 0;
this.reason = this.buildReason('operator-reset', now);
}
getSnapshot(): DependencyCircuitSnapshot {
const now = this.now();
this.prune(now);
return this.snapshot(now);
}
private evaluateClosed(now: number): void {
if (this.stateValue !== 'closed') return;
const metrics = this.metrics();
if (metrics.sampleCount < this.policy.minimumSamples) return;
if (metrics.failureRate >= this.policy.failureRateThreshold) {
this.open('failure-rate', now);
return;
}
if (metrics.slowCallRate >= this.policy.slowCallRateThreshold) {
this.open('slow-call-rate', now);
}
}
private open(
code: Extract<
DependencyCircuitReason['code'],
'failure-rate' | 'slow-call-rate' | 'probe-failed'
>,
now: number
): void {
const jitterFactor =
1 + (Math.min(Math.max(this.jitter(), 0), 1) * 2 - 1) * this.policy.openDurationJitterRatio;
this.stateValue = 'open';
this.openedAt = now;
this.nextProbeAt = now + Math.round(this.policy.openDurationMs * jitterFactor);
this.halfOpenSuccesses = 0;
this.reason = this.buildReason(code, now);
}
private close(code: 'probe-succeeded', now: number): void {
this.stateValue = 'closed';
this.samples = [];
this.openedAt = undefined;
this.nextProbeAt = undefined;
this.halfOpenSuccesses = 0;
this.reason = this.buildReason(code, now);
}
private prune(now: number): void {
const cutoff = now - this.policy.rollingWindowMs;
this.samples = this.samples.filter((sample) => sample.occurredAt > cutoff);
}
private activeProbeCount(): number {
return [...this.leases.values()].filter((lease) => lease.lease.probe && !lease.settled).length;
}
private metrics() {
const sampleCount = this.samples.length;
const failureCount = this.samples.filter((sample) => isFailure(sample.outcome)).length;
const slowCallCount = this.samples.filter(
(sample) =>
sample.outcome === 'slow-success' || sample.durationMs >= this.policy.slowCallDurationMs
).length;
return {
sampleCount,
failureCount,
slowCallCount,
failureRate: sampleCount === 0 ? 0 : failureCount / sampleCount,
slowCallRate: sampleCount === 0 ? 0 : slowCallCount / sampleCount,
};
}
private buildReason(code: DependencyCircuitReason['code'], now: number): DependencyCircuitReason {
return {
code,
observedAt: new Date(now).toISOString(),
...this.metrics(),
};
}
private snapshot(now: number): DependencyCircuitSnapshot {
const metrics = this.metrics();
return {
schemaVersion: DEPENDENCY_CIRCUIT_SCHEMA_VERSION,
key: this.key,
dependency: this.dependency,
policy: this.policy,
state: this.stateValue,
reason: this.reason,
...metrics,
openedAt: this.openedAt ? new Date(this.openedAt).toISOString() : undefined,
nextProbeAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
halfOpenInFlight: this.activeProbeCount(),
halfOpenSuccesses: this.halfOpenSuccesses,
lastOutcome: this.lastOutcome,
lastOutcomeAt: this.lastOutcomeAt
? new Date(this.lastOutcomeAt).toISOString()
: undefined,
updatedAt: new Date(now).toISOString(),
};
}
}

View file

@ -0,0 +1,123 @@
export const DEPENDENCY_CIRCUIT_SCHEMA_VERSION = 'dependency-circuit/v1' as const;
export const DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION =
'dependency-circuit-policy/v1' as const;
export const DEPENDENCY_KINDS = [
'provider',
'model-endpoint',
'agent-host',
'mcp-server',
'tool-server',
'integration',
'storage',
] as const;
export const DEPENDENCY_OUTCOMES = [
'success',
'slow-success',
'dependency-failure',
'caller-cancellation',
'policy-block',
'validation-error',
'timeout',
'throttled',
'overload',
] as const;
export const DEPENDENCY_CIRCUIT_STATES = ['closed', 'open', 'half-open'] as const;
export type DependencyKind = (typeof DEPENDENCY_KINDS)[number];
export type DependencyOutcome = (typeof DEPENDENCY_OUTCOMES)[number];
export type DependencyCircuitState = (typeof DEPENDENCY_CIRCUIT_STATES)[number];
export interface DependencyIdentity {
kind: DependencyKind;
id: string;
workspaceId?: string;
provider?: string;
model?: string;
hostId?: string;
}
export interface DependencyCircuitPolicy {
schemaVersion: typeof DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION;
minimumSamples: number;
rollingWindowMs: number;
failureRateThreshold: number;
slowCallDurationMs: number;
slowCallRateThreshold: number;
openDurationMs: number;
openDurationJitterRatio: number;
halfOpenMaxConcurrent: number;
probeSuccessThreshold: number;
}
export interface DependencyCircuitReason {
code:
| 'failure-rate'
| 'slow-call-rate'
| 'probe-failed'
| 'operator-reset'
| 'probe-window-opened'
| 'probe-succeeded';
observedAt: string;
sampleCount: number;
failureCount: number;
slowCallCount: number;
failureRate: number;
slowCallRate: number;
}
export interface DependencyCircuitSnapshot {
schemaVersion: typeof DEPENDENCY_CIRCUIT_SCHEMA_VERSION;
key: string;
dependency: DependencyIdentity;
policy: DependencyCircuitPolicy;
state: DependencyCircuitState;
reason?: DependencyCircuitReason;
sampleCount: number;
failureCount: number;
slowCallCount: number;
failureRate: number;
slowCallRate: number;
openedAt?: string;
nextProbeAt?: string;
halfOpenInFlight: number;
halfOpenSuccesses: number;
lastOutcome?: DependencyOutcome;
lastOutcomeAt?: string;
updatedAt: string;
}
export interface DependencyCircuitLease {
id: string;
circuitKey: string;
probe: boolean;
acquiredAt: string;
}
export type DependencyCircuitAdmission =
| {
allowed: true;
decision: 'allow' | 'probe';
lease: DependencyCircuitLease;
snapshot: DependencyCircuitSnapshot;
}
| {
allowed: false;
decision: 'reject';
reason: 'circuit-open' | 'probe-concurrency-exhausted';
retryAt?: string;
snapshot: DependencyCircuitSnapshot;
};
export interface DependencyOutcomeSignals {
succeeded?: boolean;
durationMs?: number;
callerCancelled?: boolean;
policyBlocked?: boolean;
validationFailed?: boolean;
timedOut?: boolean;
statusCode?: number;
errorCode?: string;
}

View file

@ -62,6 +62,7 @@ export * from './run-launch-manifest.types.js';
export * from './run-event.types.js';
export * from './progress-watchdog.types.js';
export * from './run-output-artifact.types.js';
export * from './dependency-circuit.types.js';
export * from './run-recovery.types.js';
export * from './runtime-hook.types.js';
export * from './auth.types.js';