mirror of
https://github.com/BradGroux/veritas-kanban.git
synced 2026-08-28 02:44:59 +00:00
feat: define dependency circuit control semantics (#1097)
This commit is contained in:
parent
aa17223a06
commit
1e6869613b
5 changed files with 732 additions and 0 deletions
256
server/src/__tests__/dependency-circuit-breaker.test.ts
Normal file
256
server/src/__tests__/dependency-circuit-breaker.test.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import type {
|
||||
DependencyCircuitAdmission,
|
||||
DependencyCircuitPolicy,
|
||||
DependencyOutcome,
|
||||
} from '@veritas-kanban/shared';
|
||||
import {
|
||||
DependencyCircuitBreaker,
|
||||
classifyDependencyOutcome,
|
||||
dependencyCircuitKey,
|
||||
opaqueDependencyId,
|
||||
} from '../services/dependency-circuit-breaker.js';
|
||||
|
||||
function policy(overrides: Partial<DependencyCircuitPolicy> = {}): DependencyCircuitPolicy {
|
||||
return {
|
||||
schemaVersion: 'dependency-circuit-policy/v1',
|
||||
minimumSamples: 4,
|
||||
rollingWindowMs: 10_000,
|
||||
failureRateThreshold: 0.5,
|
||||
slowCallDurationMs: 1_000,
|
||||
slowCallRateThreshold: 0.5,
|
||||
openDurationMs: 2_000,
|
||||
openDurationJitterRatio: 0,
|
||||
halfOpenMaxConcurrent: 1,
|
||||
probeSuccessThreshold: 2,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function fixture(overrides: Partial<DependencyCircuitPolicy> = {}) {
|
||||
let now = Date.parse('2026-07-25T12:00:00.000Z');
|
||||
const breaker = new DependencyCircuitBreaker({
|
||||
dependency: {
|
||||
kind: 'model-endpoint',
|
||||
id: 'openai-gpt',
|
||||
workspaceId: 'workspace_1',
|
||||
provider: 'codex-cli',
|
||||
model: 'gpt-5.6',
|
||||
},
|
||||
policy: policy(overrides),
|
||||
now: () => now,
|
||||
jitter: () => 0.5,
|
||||
});
|
||||
return {
|
||||
breaker,
|
||||
advance: (milliseconds: number) => {
|
||||
now += milliseconds;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function admitted(admission: DependencyCircuitAdmission) {
|
||||
if (!admission.allowed) throw new Error('Expected the circuit to admit the call.');
|
||||
return admission.lease;
|
||||
}
|
||||
|
||||
function record(
|
||||
breaker: DependencyCircuitBreaker,
|
||||
outcome: DependencyOutcome,
|
||||
durationMs = 10
|
||||
) {
|
||||
const lease = admitted(breaker.acquire());
|
||||
breaker.record(lease, outcome, durationMs);
|
||||
}
|
||||
|
||||
describe('DependencyCircuitBreaker', () => {
|
||||
it('builds a stable scoped key while offering a hash for secret endpoint identities', () => {
|
||||
const identity = {
|
||||
kind: 'provider' as const,
|
||||
id: opaqueDependencyId('https://secret-host.example/v1?token=do-not-store'),
|
||||
workspaceId: 'workspace_1',
|
||||
provider: 'codex-cli',
|
||||
};
|
||||
|
||||
expect(identity.id).toMatch(/^dep_[A-Za-z0-9_-]{24}$/);
|
||||
expect(identity.id).not.toContain('secret-host');
|
||||
expect(dependencyCircuitKey(identity)).toBe(
|
||||
`workspace_1:provider:${identity.id}:codex-cli:-:-`
|
||||
);
|
||||
});
|
||||
|
||||
it('requires the minimum sample size before evaluating failure rate', () => {
|
||||
const { breaker } = fixture({ minimumSamples: 3, failureRateThreshold: 0.5 });
|
||||
|
||||
record(breaker, 'dependency-failure');
|
||||
record(breaker, 'dependency-failure');
|
||||
expect(breaker.getSnapshot().state).toBe('closed');
|
||||
record(breaker, 'success');
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'open',
|
||||
reason: {
|
||||
code: 'failure-rate',
|
||||
sampleCount: 3,
|
||||
failureCount: 2,
|
||||
failureRate: 2 / 3,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('opens exactly at the configured failure-rate boundary', () => {
|
||||
const { breaker } = fixture();
|
||||
|
||||
record(breaker, 'success');
|
||||
record(breaker, 'dependency-failure');
|
||||
record(breaker, 'success');
|
||||
record(breaker, 'timeout');
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'open',
|
||||
failureCount: 2,
|
||||
failureRate: 0.5,
|
||||
reason: { code: 'failure-rate' },
|
||||
});
|
||||
});
|
||||
|
||||
it('excludes caller cancellation, policy denial, and validation errors', () => {
|
||||
const { breaker } = fixture({ minimumSamples: 2 });
|
||||
|
||||
record(breaker, 'caller-cancellation');
|
||||
record(breaker, 'policy-block');
|
||||
record(breaker, 'validation-error');
|
||||
record(breaker, 'success');
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'closed',
|
||||
sampleCount: 1,
|
||||
failureCount: 0,
|
||||
lastOutcome: 'success',
|
||||
});
|
||||
});
|
||||
|
||||
it('opens on slow-call rate without treating slow success as dependency failure', () => {
|
||||
const { breaker } = fixture();
|
||||
|
||||
record(breaker, 'success', 1_000);
|
||||
record(breaker, 'success', 10);
|
||||
record(breaker, 'success', 1_500);
|
||||
record(breaker, 'success', 20);
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'open',
|
||||
failureCount: 0,
|
||||
slowCallCount: 2,
|
||||
slowCallRate: 0.5,
|
||||
reason: { code: 'slow-call-rate' },
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects while open and single-flights half-open probes', () => {
|
||||
const { breaker, advance } = fixture({ minimumSamples: 1 });
|
||||
record(breaker, 'dependency-failure');
|
||||
|
||||
expect(breaker.acquire()).toMatchObject({
|
||||
allowed: false,
|
||||
reason: 'circuit-open',
|
||||
retryAt: '2026-07-25T12:00:02.000Z',
|
||||
});
|
||||
advance(2_000);
|
||||
const probe = breaker.acquire();
|
||||
expect(probe).toMatchObject({ allowed: true, decision: 'probe' });
|
||||
expect(breaker.acquire()).toMatchObject({
|
||||
allowed: false,
|
||||
reason: 'probe-concurrency-exhausted',
|
||||
});
|
||||
});
|
||||
|
||||
it('closes only after the configured number of successful probes', () => {
|
||||
const { breaker, advance } = fixture({ minimumSamples: 1 });
|
||||
record(breaker, 'dependency-failure');
|
||||
advance(2_000);
|
||||
|
||||
record(breaker, 'success');
|
||||
expect(breaker.getSnapshot()).toMatchObject({ state: 'half-open', halfOpenSuccesses: 1 });
|
||||
record(breaker, 'success');
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'closed',
|
||||
sampleCount: 0,
|
||||
reason: { code: 'probe-succeeded' },
|
||||
});
|
||||
});
|
||||
|
||||
it('reopens after a failed probe and applies bounded jitter', () => {
|
||||
const { breaker, advance } = fixture({
|
||||
minimumSamples: 1,
|
||||
openDurationJitterRatio: 0.25,
|
||||
});
|
||||
record(breaker, 'dependency-failure');
|
||||
advance(2_000);
|
||||
const probe = admitted(breaker.acquire());
|
||||
breaker.record(probe, 'overload', 20);
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'open',
|
||||
reason: { code: 'probe-failed' },
|
||||
nextProbeAt: '2026-07-25T12:00:04.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not let a stale concurrent probe close a reopened circuit', () => {
|
||||
const { breaker, advance } = fixture({
|
||||
minimumSamples: 1,
|
||||
halfOpenMaxConcurrent: 2,
|
||||
probeSuccessThreshold: 1,
|
||||
});
|
||||
record(breaker, 'dependency-failure');
|
||||
advance(2_000);
|
||||
const failed = admitted(breaker.acquire());
|
||||
const stale = admitted(breaker.acquire());
|
||||
breaker.record(failed, 'timeout', 2_000);
|
||||
breaker.record(stale, 'success', 10);
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'open',
|
||||
reason: { code: 'probe-failed' },
|
||||
});
|
||||
});
|
||||
|
||||
it('prunes old samples from the rolling window', () => {
|
||||
const { breaker, advance } = fixture({ minimumSamples: 2, rollingWindowMs: 1_000 });
|
||||
record(breaker, 'dependency-failure');
|
||||
advance(1_001);
|
||||
record(breaker, 'success');
|
||||
|
||||
expect(breaker.getSnapshot()).toMatchObject({
|
||||
state: 'closed',
|
||||
sampleCount: 1,
|
||||
failureCount: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('rejects duplicate lease settlement', () => {
|
||||
const { breaker } = fixture();
|
||||
const lease = admitted(breaker.acquire());
|
||||
breaker.record(lease, 'success', 10);
|
||||
|
||||
expect(() => breaker.record(lease, 'success', 10)).toThrow('missing, settled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('classifyDependencyOutcome', () => {
|
||||
it.each([
|
||||
[{ succeeded: true }, 'success'],
|
||||
[{ callerCancelled: true, timedOut: true }, 'caller-cancellation'],
|
||||
[{ policyBlocked: true }, 'policy-block'],
|
||||
[{ validationFailed: true }, 'validation-error'],
|
||||
[{ timedOut: true }, 'timeout'],
|
||||
[{ statusCode: 429 }, 'throttled'],
|
||||
[{ statusCode: 503 }, 'overload'],
|
||||
[{ errorCode: 'OVERLOADED' }, 'overload'],
|
||||
[{}, 'dependency-failure'],
|
||||
] as const)('classifies %j as %s', (signals, expected) => {
|
||||
expect(classifyDependencyOutcome(signals)).toBe(expected);
|
||||
});
|
||||
});
|
||||
35
server/src/schemas/dependency-circuit-schemas.ts
Normal file
35
server/src/schemas/dependency-circuit-schemas.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { z } from 'zod';
|
||||
import {
|
||||
DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
|
||||
DEPENDENCY_KINDS,
|
||||
type DependencyCircuitPolicy,
|
||||
type DependencyIdentity,
|
||||
} from '@veritas-kanban/shared';
|
||||
|
||||
const IdentifierSchema = z.string().trim().min(1).max(240);
|
||||
|
||||
export const DependencyIdentitySchema: z.ZodType<DependencyIdentity> = z
|
||||
.object({
|
||||
kind: z.enum(DEPENDENCY_KINDS),
|
||||
id: IdentifierSchema.regex(/^[A-Za-z0-9][A-Za-z0-9._:-]*$/),
|
||||
workspaceId: IdentifierSchema.optional(),
|
||||
provider: IdentifierSchema.optional(),
|
||||
model: IdentifierSchema.optional(),
|
||||
hostId: IdentifierSchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const DependencyCircuitPolicySchema: z.ZodType<DependencyCircuitPolicy> = z
|
||||
.object({
|
||||
schemaVersion: z.literal(DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION),
|
||||
minimumSamples: z.number().int().min(1).max(100_000),
|
||||
rollingWindowMs: z.number().int().min(1_000).max(24 * 60 * 60 * 1_000),
|
||||
failureRateThreshold: z.number().min(0.01).max(1),
|
||||
slowCallDurationMs: z.number().int().min(1).max(60 * 60 * 1_000),
|
||||
slowCallRateThreshold: z.number().min(0.01).max(1),
|
||||
openDurationMs: z.number().int().min(100).max(60 * 60 * 1_000),
|
||||
openDurationJitterRatio: z.number().min(0).max(0.5),
|
||||
halfOpenMaxConcurrent: z.number().int().min(1).max(1_000),
|
||||
probeSuccessThreshold: z.number().int().min(1).max(10_000),
|
||||
})
|
||||
.strict();
|
||||
317
server/src/services/dependency-circuit-breaker.ts
Normal file
317
server/src/services/dependency-circuit-breaker.ts
Normal file
|
|
@ -0,0 +1,317 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { nanoid } from 'nanoid';
|
||||
import {
|
||||
DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
|
||||
DEPENDENCY_CIRCUIT_SCHEMA_VERSION,
|
||||
type DependencyCircuitAdmission,
|
||||
type DependencyCircuitLease,
|
||||
type DependencyCircuitPolicy,
|
||||
type DependencyCircuitReason,
|
||||
type DependencyCircuitSnapshot,
|
||||
type DependencyIdentity,
|
||||
type DependencyOutcome,
|
||||
type DependencyOutcomeSignals,
|
||||
} from '@veritas-kanban/shared';
|
||||
import {
|
||||
DependencyCircuitPolicySchema,
|
||||
DependencyIdentitySchema,
|
||||
} from '../schemas/dependency-circuit-schemas.js';
|
||||
|
||||
export const DEFAULT_DEPENDENCY_CIRCUIT_POLICY: DependencyCircuitPolicy = {
|
||||
schemaVersion: DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION,
|
||||
minimumSamples: 10,
|
||||
rollingWindowMs: 60_000,
|
||||
failureRateThreshold: 0.5,
|
||||
slowCallDurationMs: 30_000,
|
||||
slowCallRateThreshold: 0.5,
|
||||
openDurationMs: 30_000,
|
||||
openDurationJitterRatio: 0.1,
|
||||
halfOpenMaxConcurrent: 1,
|
||||
probeSuccessThreshold: 2,
|
||||
};
|
||||
|
||||
interface CircuitSample {
|
||||
occurredAt: number;
|
||||
outcome: DependencyOutcome;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
interface ActiveLease {
|
||||
lease: DependencyCircuitLease;
|
||||
settled: boolean;
|
||||
}
|
||||
|
||||
export interface DependencyCircuitBreakerOptions {
|
||||
dependency: DependencyIdentity;
|
||||
policy?: DependencyCircuitPolicy;
|
||||
now?: () => number;
|
||||
jitter?: () => number;
|
||||
}
|
||||
|
||||
function isFailure(outcome: DependencyOutcome): boolean {
|
||||
return ['dependency-failure', 'timeout', 'throttled', 'overload'].includes(outcome);
|
||||
}
|
||||
|
||||
function isExcluded(outcome: DependencyOutcome): boolean {
|
||||
return ['caller-cancellation', 'policy-block', 'validation-error'].includes(outcome);
|
||||
}
|
||||
|
||||
export function dependencyCircuitKey(identity: DependencyIdentity): string {
|
||||
const parsed = DependencyIdentitySchema.parse(identity);
|
||||
return [
|
||||
parsed.workspaceId ?? 'global',
|
||||
parsed.kind,
|
||||
parsed.id,
|
||||
parsed.provider ?? '-',
|
||||
parsed.model ?? '-',
|
||||
parsed.hostId ?? '-',
|
||||
].join(':');
|
||||
}
|
||||
|
||||
export function opaqueDependencyId(value: string): string {
|
||||
return `dep_${createHash('sha256').update(value).digest('base64url').slice(0, 24)}`;
|
||||
}
|
||||
|
||||
export function classifyDependencyOutcome(signals: DependencyOutcomeSignals): DependencyOutcome {
|
||||
if (signals.callerCancelled) return 'caller-cancellation';
|
||||
if (signals.policyBlocked) return 'policy-block';
|
||||
if (signals.validationFailed) return 'validation-error';
|
||||
if (signals.timedOut || signals.errorCode === 'ETIMEDOUT') return 'timeout';
|
||||
if (signals.statusCode === 429 || signals.errorCode === 'RATE_LIMITED') return 'throttled';
|
||||
if (
|
||||
signals.statusCode === 503 ||
|
||||
signals.statusCode === 529 ||
|
||||
signals.errorCode === 'OVERLOADED'
|
||||
) {
|
||||
return 'overload';
|
||||
}
|
||||
if (signals.succeeded) return 'success';
|
||||
return 'dependency-failure';
|
||||
}
|
||||
|
||||
export class DependencyCircuitBreaker {
|
||||
readonly key: string;
|
||||
readonly dependency: DependencyIdentity;
|
||||
readonly policy: DependencyCircuitPolicy;
|
||||
|
||||
private readonly now: () => number;
|
||||
private readonly jitter: () => number;
|
||||
private stateValue: DependencyCircuitSnapshot['state'] = 'closed';
|
||||
private samples: CircuitSample[] = [];
|
||||
private leases = new Map<string, ActiveLease>();
|
||||
private reason?: DependencyCircuitReason;
|
||||
private openedAt?: number;
|
||||
private nextProbeAt?: number;
|
||||
private halfOpenSuccesses = 0;
|
||||
private lastOutcome?: DependencyOutcome;
|
||||
private lastOutcomeAt?: number;
|
||||
|
||||
constructor(options: DependencyCircuitBreakerOptions) {
|
||||
this.dependency = DependencyIdentitySchema.parse(options.dependency);
|
||||
this.policy = DependencyCircuitPolicySchema.parse(
|
||||
options.policy ?? DEFAULT_DEPENDENCY_CIRCUIT_POLICY
|
||||
);
|
||||
this.key = dependencyCircuitKey(this.dependency);
|
||||
this.now = options.now ?? Date.now;
|
||||
this.jitter = options.jitter ?? Math.random;
|
||||
}
|
||||
|
||||
acquire(): DependencyCircuitAdmission {
|
||||
const now = this.now();
|
||||
this.prune(now);
|
||||
if (
|
||||
this.stateValue === 'open' &&
|
||||
this.nextProbeAt !== undefined &&
|
||||
now >= this.nextProbeAt
|
||||
) {
|
||||
this.stateValue = 'half-open';
|
||||
this.halfOpenSuccesses = 0;
|
||||
this.reason = this.buildReason('probe-window-opened', now);
|
||||
}
|
||||
if (this.stateValue === 'open') {
|
||||
return {
|
||||
allowed: false,
|
||||
decision: 'reject',
|
||||
reason: 'circuit-open',
|
||||
retryAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
|
||||
snapshot: this.snapshot(now),
|
||||
};
|
||||
}
|
||||
const probe = this.stateValue === 'half-open';
|
||||
if (probe && this.activeProbeCount() >= this.policy.halfOpenMaxConcurrent) {
|
||||
return {
|
||||
allowed: false,
|
||||
decision: 'reject',
|
||||
reason: 'probe-concurrency-exhausted',
|
||||
retryAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
|
||||
snapshot: this.snapshot(now),
|
||||
};
|
||||
}
|
||||
const lease: DependencyCircuitLease = {
|
||||
id: `cirlease_${nanoid(18)}`,
|
||||
circuitKey: this.key,
|
||||
probe,
|
||||
acquiredAt: new Date(now).toISOString(),
|
||||
};
|
||||
this.leases.set(lease.id, { lease, settled: false });
|
||||
return {
|
||||
allowed: true,
|
||||
decision: probe ? 'probe' : 'allow',
|
||||
lease,
|
||||
snapshot: this.snapshot(now),
|
||||
};
|
||||
}
|
||||
|
||||
record(lease: DependencyCircuitLease, outcome: DependencyOutcome, durationMs: number): void {
|
||||
const active = this.leases.get(lease.id);
|
||||
if (!active || active.settled || active.lease.circuitKey !== this.key) {
|
||||
throw new Error('Dependency circuit lease is missing, settled, or belongs to another circuit.');
|
||||
}
|
||||
if (!Number.isFinite(durationMs) || durationMs < 0) {
|
||||
throw new Error('Dependency circuit duration must be a non-negative finite number.');
|
||||
}
|
||||
active.settled = true;
|
||||
this.leases.delete(lease.id);
|
||||
const now = this.now();
|
||||
this.lastOutcome = outcome;
|
||||
this.lastOutcomeAt = now;
|
||||
|
||||
if (!isExcluded(outcome)) {
|
||||
const normalizedOutcome =
|
||||
outcome === 'success' && durationMs >= this.policy.slowCallDurationMs
|
||||
? 'slow-success'
|
||||
: outcome;
|
||||
this.samples.push({ occurredAt: now, outcome: normalizedOutcome, durationMs });
|
||||
this.lastOutcome = normalizedOutcome;
|
||||
}
|
||||
this.prune(now);
|
||||
|
||||
if (!lease.probe) {
|
||||
this.evaluateClosed(now);
|
||||
return;
|
||||
}
|
||||
if (this.stateValue !== 'half-open') return;
|
||||
if (isExcluded(outcome)) return;
|
||||
const failed =
|
||||
isFailure(this.lastOutcome ?? outcome) ||
|
||||
durationMs >= this.policy.slowCallDurationMs;
|
||||
if (failed) {
|
||||
this.open('probe-failed', now);
|
||||
return;
|
||||
}
|
||||
this.halfOpenSuccesses += 1;
|
||||
if (this.halfOpenSuccesses >= this.policy.probeSuccessThreshold) {
|
||||
this.close('probe-succeeded', now);
|
||||
}
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
const now = this.now();
|
||||
this.samples = [];
|
||||
this.leases.clear();
|
||||
this.stateValue = 'closed';
|
||||
this.openedAt = undefined;
|
||||
this.nextProbeAt = undefined;
|
||||
this.halfOpenSuccesses = 0;
|
||||
this.reason = this.buildReason('operator-reset', now);
|
||||
}
|
||||
|
||||
getSnapshot(): DependencyCircuitSnapshot {
|
||||
const now = this.now();
|
||||
this.prune(now);
|
||||
return this.snapshot(now);
|
||||
}
|
||||
|
||||
private evaluateClosed(now: number): void {
|
||||
if (this.stateValue !== 'closed') return;
|
||||
const metrics = this.metrics();
|
||||
if (metrics.sampleCount < this.policy.minimumSamples) return;
|
||||
if (metrics.failureRate >= this.policy.failureRateThreshold) {
|
||||
this.open('failure-rate', now);
|
||||
return;
|
||||
}
|
||||
if (metrics.slowCallRate >= this.policy.slowCallRateThreshold) {
|
||||
this.open('slow-call-rate', now);
|
||||
}
|
||||
}
|
||||
|
||||
private open(
|
||||
code: Extract<
|
||||
DependencyCircuitReason['code'],
|
||||
'failure-rate' | 'slow-call-rate' | 'probe-failed'
|
||||
>,
|
||||
now: number
|
||||
): void {
|
||||
const jitterFactor =
|
||||
1 + (Math.min(Math.max(this.jitter(), 0), 1) * 2 - 1) * this.policy.openDurationJitterRatio;
|
||||
this.stateValue = 'open';
|
||||
this.openedAt = now;
|
||||
this.nextProbeAt = now + Math.round(this.policy.openDurationMs * jitterFactor);
|
||||
this.halfOpenSuccesses = 0;
|
||||
this.reason = this.buildReason(code, now);
|
||||
}
|
||||
|
||||
private close(code: 'probe-succeeded', now: number): void {
|
||||
this.stateValue = 'closed';
|
||||
this.samples = [];
|
||||
this.openedAt = undefined;
|
||||
this.nextProbeAt = undefined;
|
||||
this.halfOpenSuccesses = 0;
|
||||
this.reason = this.buildReason(code, now);
|
||||
}
|
||||
|
||||
private prune(now: number): void {
|
||||
const cutoff = now - this.policy.rollingWindowMs;
|
||||
this.samples = this.samples.filter((sample) => sample.occurredAt > cutoff);
|
||||
}
|
||||
|
||||
private activeProbeCount(): number {
|
||||
return [...this.leases.values()].filter((lease) => lease.lease.probe && !lease.settled).length;
|
||||
}
|
||||
|
||||
private metrics() {
|
||||
const sampleCount = this.samples.length;
|
||||
const failureCount = this.samples.filter((sample) => isFailure(sample.outcome)).length;
|
||||
const slowCallCount = this.samples.filter(
|
||||
(sample) =>
|
||||
sample.outcome === 'slow-success' || sample.durationMs >= this.policy.slowCallDurationMs
|
||||
).length;
|
||||
return {
|
||||
sampleCount,
|
||||
failureCount,
|
||||
slowCallCount,
|
||||
failureRate: sampleCount === 0 ? 0 : failureCount / sampleCount,
|
||||
slowCallRate: sampleCount === 0 ? 0 : slowCallCount / sampleCount,
|
||||
};
|
||||
}
|
||||
|
||||
private buildReason(code: DependencyCircuitReason['code'], now: number): DependencyCircuitReason {
|
||||
return {
|
||||
code,
|
||||
observedAt: new Date(now).toISOString(),
|
||||
...this.metrics(),
|
||||
};
|
||||
}
|
||||
|
||||
private snapshot(now: number): DependencyCircuitSnapshot {
|
||||
const metrics = this.metrics();
|
||||
return {
|
||||
schemaVersion: DEPENDENCY_CIRCUIT_SCHEMA_VERSION,
|
||||
key: this.key,
|
||||
dependency: this.dependency,
|
||||
policy: this.policy,
|
||||
state: this.stateValue,
|
||||
reason: this.reason,
|
||||
...metrics,
|
||||
openedAt: this.openedAt ? new Date(this.openedAt).toISOString() : undefined,
|
||||
nextProbeAt: this.nextProbeAt ? new Date(this.nextProbeAt).toISOString() : undefined,
|
||||
halfOpenInFlight: this.activeProbeCount(),
|
||||
halfOpenSuccesses: this.halfOpenSuccesses,
|
||||
lastOutcome: this.lastOutcome,
|
||||
lastOutcomeAt: this.lastOutcomeAt
|
||||
? new Date(this.lastOutcomeAt).toISOString()
|
||||
: undefined,
|
||||
updatedAt: new Date(now).toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
123
shared/src/types/dependency-circuit.types.ts
Normal file
123
shared/src/types/dependency-circuit.types.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
export const DEPENDENCY_CIRCUIT_SCHEMA_VERSION = 'dependency-circuit/v1' as const;
|
||||
export const DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION =
|
||||
'dependency-circuit-policy/v1' as const;
|
||||
|
||||
export const DEPENDENCY_KINDS = [
|
||||
'provider',
|
||||
'model-endpoint',
|
||||
'agent-host',
|
||||
'mcp-server',
|
||||
'tool-server',
|
||||
'integration',
|
||||
'storage',
|
||||
] as const;
|
||||
|
||||
export const DEPENDENCY_OUTCOMES = [
|
||||
'success',
|
||||
'slow-success',
|
||||
'dependency-failure',
|
||||
'caller-cancellation',
|
||||
'policy-block',
|
||||
'validation-error',
|
||||
'timeout',
|
||||
'throttled',
|
||||
'overload',
|
||||
] as const;
|
||||
|
||||
export const DEPENDENCY_CIRCUIT_STATES = ['closed', 'open', 'half-open'] as const;
|
||||
|
||||
export type DependencyKind = (typeof DEPENDENCY_KINDS)[number];
|
||||
export type DependencyOutcome = (typeof DEPENDENCY_OUTCOMES)[number];
|
||||
export type DependencyCircuitState = (typeof DEPENDENCY_CIRCUIT_STATES)[number];
|
||||
|
||||
export interface DependencyIdentity {
|
||||
kind: DependencyKind;
|
||||
id: string;
|
||||
workspaceId?: string;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
hostId?: string;
|
||||
}
|
||||
|
||||
export interface DependencyCircuitPolicy {
|
||||
schemaVersion: typeof DEPENDENCY_CIRCUIT_POLICY_SCHEMA_VERSION;
|
||||
minimumSamples: number;
|
||||
rollingWindowMs: number;
|
||||
failureRateThreshold: number;
|
||||
slowCallDurationMs: number;
|
||||
slowCallRateThreshold: number;
|
||||
openDurationMs: number;
|
||||
openDurationJitterRatio: number;
|
||||
halfOpenMaxConcurrent: number;
|
||||
probeSuccessThreshold: number;
|
||||
}
|
||||
|
||||
export interface DependencyCircuitReason {
|
||||
code:
|
||||
| 'failure-rate'
|
||||
| 'slow-call-rate'
|
||||
| 'probe-failed'
|
||||
| 'operator-reset'
|
||||
| 'probe-window-opened'
|
||||
| 'probe-succeeded';
|
||||
observedAt: string;
|
||||
sampleCount: number;
|
||||
failureCount: number;
|
||||
slowCallCount: number;
|
||||
failureRate: number;
|
||||
slowCallRate: number;
|
||||
}
|
||||
|
||||
export interface DependencyCircuitSnapshot {
|
||||
schemaVersion: typeof DEPENDENCY_CIRCUIT_SCHEMA_VERSION;
|
||||
key: string;
|
||||
dependency: DependencyIdentity;
|
||||
policy: DependencyCircuitPolicy;
|
||||
state: DependencyCircuitState;
|
||||
reason?: DependencyCircuitReason;
|
||||
sampleCount: number;
|
||||
failureCount: number;
|
||||
slowCallCount: number;
|
||||
failureRate: number;
|
||||
slowCallRate: number;
|
||||
openedAt?: string;
|
||||
nextProbeAt?: string;
|
||||
halfOpenInFlight: number;
|
||||
halfOpenSuccesses: number;
|
||||
lastOutcome?: DependencyOutcome;
|
||||
lastOutcomeAt?: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface DependencyCircuitLease {
|
||||
id: string;
|
||||
circuitKey: string;
|
||||
probe: boolean;
|
||||
acquiredAt: string;
|
||||
}
|
||||
|
||||
export type DependencyCircuitAdmission =
|
||||
| {
|
||||
allowed: true;
|
||||
decision: 'allow' | 'probe';
|
||||
lease: DependencyCircuitLease;
|
||||
snapshot: DependencyCircuitSnapshot;
|
||||
}
|
||||
| {
|
||||
allowed: false;
|
||||
decision: 'reject';
|
||||
reason: 'circuit-open' | 'probe-concurrency-exhausted';
|
||||
retryAt?: string;
|
||||
snapshot: DependencyCircuitSnapshot;
|
||||
};
|
||||
|
||||
export interface DependencyOutcomeSignals {
|
||||
succeeded?: boolean;
|
||||
durationMs?: number;
|
||||
callerCancelled?: boolean;
|
||||
policyBlocked?: boolean;
|
||||
validationFailed?: boolean;
|
||||
timedOut?: boolean;
|
||||
statusCode?: number;
|
||||
errorCode?: string;
|
||||
}
|
||||
|
|
@ -62,6 +62,7 @@ export * from './run-launch-manifest.types.js';
|
|||
export * from './run-event.types.js';
|
||||
export * from './progress-watchdog.types.js';
|
||||
export * from './run-output-artifact.types.js';
|
||||
export * from './dependency-circuit.types.js';
|
||||
export * from './run-recovery.types.js';
|
||||
export * from './runtime-hook.types.js';
|
||||
export * from './auth.types.js';
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue