diff --git a/gitnexus/src/core/evolver/index.ts b/gitnexus/src/core/evolver/index.ts index d4702960d..ce5cb8234 100644 --- a/gitnexus/src/core/evolver/index.ts +++ b/gitnexus/src/core/evolver/index.ts @@ -1 +1,2 @@ export * from './types.js'; +export * from './metric-evaluator.js'; diff --git a/gitnexus/src/core/evolver/metric-evaluator.ts b/gitnexus/src/core/evolver/metric-evaluator.ts new file mode 100644 index 000000000..60114879e --- /dev/null +++ b/gitnexus/src/core/evolver/metric-evaluator.ts @@ -0,0 +1,159 @@ +import type { + BaselineProfile, + EvaluationReport, + EvaluationVerdict, + MetricDelta, + MetricDirection, + MetricReading, + MetricTarget, + RegressionFinding, + ResourceBudget, + ResourceUsage, + SafetyFinding, + TrialResult, + VariantSpec, +} from './types.js'; + +export interface EvaluateVariantInput { + id: string; + variant: VariantSpec; + baseline: BaselineProfile; + trial: TrialResult; + targetMetrics: MetricTarget[]; + budget: ResourceBudget; + safetyFindings: SafetyFinding[]; + rollbackPassed: boolean; +} + +export function evaluateVariant(input: EvaluateVariantInput): EvaluationReport { + const metricDeltas = input.targetMetrics.map((target) => + calculateMetricDelta(target, input.baseline.metrics, input.trial.metrics), + ); + const regressions = input.targetMetrics.flatMap((target) => + findRegression(target, input.baseline.metrics, input.trial.metrics), + ); + const budgetPassed = isWithinBudget(input.trial.resourceUsage, input.budget); + const metricsPassed = metricDeltas.every((delta) => delta.passed); + const checksPassed = + metricsPassed && + regressions.length === 0 && + budgetPassed && + input.safetyFindings.length === 0 && + input.rollbackPassed; + + return { + id: input.id, + variantId: input.variant.id, + baselineId: input.baseline.id, + metricDeltas, + regressions, + resourceCost: input.trial.resourceUsage, + safetyFindings: input.safetyFindings, + verdict: determineVerdict(input.variant, checksPassed), + }; +} + +function determineVerdict(variant: VariantSpec, checksPassed: boolean): EvaluationVerdict { + if (!checksPassed) { + return 'reject'; + } + + if (variant.mutationType === 'parameter') { + return 'promote'; + } + + return 'needs-review'; +} + +function calculateMetricDelta( + target: MetricTarget, + baselineMetrics: MetricReading[], + trialMetrics: MetricReading[], +): MetricDelta { + const baselineValue = findMetricValue(target.metricId, baselineMetrics); + const candidateValue = findMetricValue(target.metricId, trialMetrics); + const delta = candidateValue - baselineValue; + const minDelta = target.minDelta ?? 0; + const passed = isImprovement(delta, target.direction, minDelta); + + return { + metricId: target.metricId, + baselineValue, + candidateValue, + delta: normalizeDelta(delta), + direction: target.direction, + passed, + }; +} + +function findRegression( + target: MetricTarget, + baselineMetrics: MetricReading[], + trialMetrics: MetricReading[], +): RegressionFinding[] { + const baselineValue = findMetricValue(target.metricId, baselineMetrics); + const candidateValue = findMetricValue(target.metricId, trialMetrics); + const regressionAmount = calculateRegressionAmount( + baselineValue, + candidateValue, + target.direction, + ); + const limit = target.maxRegression ?? 0; + + if (regressionAmount <= limit) { + return []; + } + + return [ + { + metricId: target.metricId, + baselineValue, + candidateValue, + regressionAmount: normalizeDelta(regressionAmount), + limit, + }, + ]; +} + +function isImprovement(delta: number, direction: MetricDirection, minDelta: number): boolean { + if (direction === 'increase') { + return delta >= minDelta; + } + + return -delta >= minDelta; +} + +function calculateRegressionAmount( + baselineValue: number, + candidateValue: number, + direction: MetricDirection, +): number { + if (direction === 'increase') { + return Math.max(0, baselineValue - candidateValue); + } + + return Math.max(0, candidateValue - baselineValue); +} + +function findMetricValue(metricId: string, metrics: MetricReading[]): number { + const metric = metrics.find((reading) => reading.metricId === metricId); + + if (!metric) { + return 0; + } + + return metric.value; +} + +function isWithinBudget(usage: ResourceUsage, budget: ResourceBudget): boolean { + return ( + usage.durationMs <= budget.maxDurationMs && + usage.computeUnits <= budget.maxComputeUnits && + usage.apiCostUsd <= budget.maxApiCostUsd && + usage.storageBytes <= budget.maxStorageBytes + ); +} + +function normalizeDelta(value: number): number { + return Number(value.toFixed(12)); +} diff --git a/gitnexus/test/unit/evolver/metric-evaluator.test.ts b/gitnexus/test/unit/evolver/metric-evaluator.test.ts new file mode 100644 index 000000000..5c13df264 --- /dev/null +++ b/gitnexus/test/unit/evolver/metric-evaluator.test.ts @@ -0,0 +1,280 @@ +import { describe, expect, it } from 'vitest'; +import { evaluateVariant } from '../../../src/core/evolver/index.js'; +import type { + BaselineProfile, + MetricTarget, + ResourceBudget, + SafetyFinding, + TrialResult, + VariantSpec, +} from '../../../src/core/evolver/index.js'; + +const capturedAt = '2026-05-25T00:00:00.000Z'; + +function baselineProfile(metricValue = 0.72): BaselineProfile { + return { + id: 'baseline-1', + goalId: 'goal-1', + capturedAt, + metrics: [ + { + metricId: 'task_success_rate', + value: metricValue, + capturedAt, + }, + ], + resourceUsage: { + durationMs: 100, + computeUnits: 1, + apiCostUsd: 0, + storageBytes: 128, + }, + }; +} + +function parameterVariant(overrides: Partial = {}): VariantSpec { + return { + id: 'variant-parameter-1', + planId: 'plan-1', + mutationType: 'parameter', + description: 'Increase retrieval topK', + patch: { + operations: [ + { + op: 'setParameter', + path: 'retrieval.topK', + previousValue: 5, + nextValue: 8, + }, + ], + }, + expectedGain: [ + { + metricId: 'task_success_rate', + expectedDelta: 0.05, + direction: 'increase', + }, + ], + riskLevel: 'low', + rollbackPlan: { + strategy: 'restore-previous-parameters', + operations: [ + { + op: 'setParameter', + path: 'retrieval.topK', + previousValue: 8, + nextValue: 5, + }, + ], + }, + provenance: [ + { + source: 'synthetic-benchmark', + reference: 'run-1', + capturedAt, + }, + ], + ...overrides, + }; +} + +function trialResult(candidateValue: number, overrides: Partial = {}): TrialResult { + return { + id: 'trial-1', + variantId: 'variant-parameter-1', + protocolId: 'protocol-1', + metrics: [ + { + metricId: 'task_success_rate', + value: candidateValue, + capturedAt, + }, + ], + resourceUsage: { + durationMs: 120, + computeUnits: 1, + apiCostUsd: 0, + storageBytes: 256, + }, + errors: [], + completedAt: capturedAt, + ...overrides, + }; +} + +const targetMetrics: MetricTarget[] = [ + { + metricId: 'task_success_rate', + direction: 'increase', + minDelta: 0.05, + maxRegression: 0.01, + }, +]; + +const budget: ResourceBudget = { + maxDurationMs: 1000, + maxComputeUnits: 10, + maxApiCostUsd: 1, + maxStorageBytes: 1024, + maxConcurrency: 1, + maxHighRiskVariants: 0, +}; + +describe('evaluateVariant', () => { + it('promotes a parameter variant when metrics improve and checks pass', () => { + const report = evaluateVariant({ + id: 'report-1', + variant: parameterVariant(), + baseline: baselineProfile(), + trial: trialResult(0.81), + targetMetrics, + budget, + safetyFindings: [], + rollbackPassed: true, + }); + + expect(report.verdict).toBe('promote'); + expect(report.metricDeltas).toEqual([ + { + metricId: 'task_success_rate', + baselineValue: 0.72, + candidateValue: 0.81, + delta: 0.09, + direction: 'increase', + passed: true, + }, + ]); + expect(report.regressions).toEqual([]); + }); + + it('rejects a variant when a target metric regresses beyond the configured limit', () => { + const report = evaluateVariant({ + id: 'report-2', + variant: parameterVariant(), + baseline: baselineProfile(), + trial: trialResult(0.68), + targetMetrics, + budget, + safetyFindings: [], + rollbackPassed: true, + }); + + expect(report.verdict).toBe('reject'); + expect(report.metricDeltas[0]?.passed).toBe(false); + expect(report.regressions).toEqual([ + { + metricId: 'task_success_rate', + baselineValue: 0.72, + candidateValue: 0.68, + regressionAmount: 0.04, + limit: 0.01, + }, + ]); + }); + + it('rejects a variant when safety findings are present', () => { + const safetyFindings: SafetyFinding[] = [ + { + id: 'safety-1', + severity: 'high', + message: 'unsafe action attempted', + }, + ]; + + const report = evaluateVariant({ + id: 'report-3', + variant: parameterVariant(), + baseline: baselineProfile(), + trial: trialResult(0.81), + targetMetrics, + budget, + safetyFindings, + rollbackPassed: true, + }); + + expect(report.verdict).toBe('reject'); + expect(report.safetyFindings).toBe(safetyFindings); + }); + + it('rejects a variant when resource cost exceeds the configured budget', () => { + const report = evaluateVariant({ + id: 'report-budget', + variant: parameterVariant(), + baseline: baselineProfile(), + trial: trialResult(0.81, { + resourceUsage: { + durationMs: 1200, + computeUnits: 1, + apiCostUsd: 0, + storageBytes: 256, + }, + }), + targetMetrics, + budget, + safetyFindings: [], + rollbackPassed: true, + }); + + expect(report.verdict).toBe('reject'); + expect(report.resourceCost.durationMs).toBe(1200); + }); + + it('rejects a variant when rollback validation fails', () => { + const report = evaluateVariant({ + id: 'report-rollback', + variant: parameterVariant(), + baseline: baselineProfile(), + trial: trialResult(0.81), + targetMetrics, + budget, + safetyFindings: [], + rollbackPassed: false, + }); + + expect(report.verdict).toBe('reject'); + expect(report.metricDeltas[0]?.passed).toBe(true); + }); + + it('marks an algorithm variant as needs-review even when metrics improve', () => { + const algorithmVariant = parameterVariant({ + id: 'variant-algorithm-1', + mutationType: 'algorithm', + riskLevel: 'medium', + patch: { + operations: [ + { + op: 'selectAlgorithm', + policyId: 'ranking-policy', + previousAlgorithm: 'baseline-ranker', + nextAlgorithm: 'candidate-ranker', + }, + ], + }, + rollbackPlan: { + strategy: 'disable-candidate', + operations: [ + { + op: 'selectAlgorithm', + policyId: 'ranking-policy', + previousAlgorithm: 'candidate-ranker', + nextAlgorithm: 'baseline-ranker', + }, + ], + }, + }); + + const report = evaluateVariant({ + id: 'report-4', + variant: algorithmVariant, + baseline: baselineProfile(), + trial: trialResult(0.83, { variantId: algorithmVariant.id }), + targetMetrics, + budget, + safetyFindings: [], + rollbackPassed: true, + }); + + expect(report.verdict).toBe('needs-review'); + expect(report.metricDeltas[0]?.passed).toBe(true); + }); +});