Add evolver metric evaluator

gitnexus/src/core/evolver/index.ts

@@ -1 +1,2 @@
 export * from './types.js';
+export * from './metric-evaluator.js';

gitnexus/src/core/evolver/metric-evaluator.ts

@@ -0,0 +1,159 @@
+import type {
+  BaselineProfile,
+  EvaluationReport,
+  EvaluationVerdict,
+  MetricDelta,
+  MetricDirection,
+  MetricReading,
+  MetricTarget,
+  RegressionFinding,
+  ResourceBudget,
+  ResourceUsage,
+  SafetyFinding,
+  TrialResult,
+  VariantSpec,
+} from './types.js';
+
+export interface EvaluateVariantInput {
+  id: string;
+  variant: VariantSpec;
+  baseline: BaselineProfile;
+  trial: TrialResult;
+  targetMetrics: MetricTarget[];
+  budget: ResourceBudget;
+  safetyFindings: SafetyFinding[];
+  rollbackPassed: boolean;
+}
+
+export function evaluateVariant(input: EvaluateVariantInput): EvaluationReport {
+  const metricDeltas = input.targetMetrics.map((target) =>
+    calculateMetricDelta(target, input.baseline.metrics, input.trial.metrics),
+  );
+  const regressions = input.targetMetrics.flatMap((target) =>
+    findRegression(target, input.baseline.metrics, input.trial.metrics),
+  );
+  const budgetPassed = isWithinBudget(input.trial.resourceUsage, input.budget);
+  const metricsPassed = metricDeltas.every((delta) => delta.passed);
+  const checksPassed =
+    metricsPassed &&
+    regressions.length === 0 &&
+    budgetPassed &&
+    input.safetyFindings.length === 0 &&
+    input.rollbackPassed;
+
+  return {
+    id: input.id,
+    variantId: input.variant.id,
+    baselineId: input.baseline.id,
+    metricDeltas,
+    regressions,
+    resourceCost: input.trial.resourceUsage,
+    safetyFindings: input.safetyFindings,
+    verdict: determineVerdict(input.variant, checksPassed),
+  };
+}
+
+function determineVerdict(variant: VariantSpec, checksPassed: boolean): EvaluationVerdict {
+  if (!checksPassed) {
+    return 'reject';
+  }
+
+  if (variant.mutationType === 'parameter') {
+    return 'promote';
+  }
+
+  return 'needs-review';
+}
+
+function calculateMetricDelta(
+  target: MetricTarget,
+  baselineMetrics: MetricReading[],
+  trialMetrics: MetricReading[],
+): MetricDelta {
+  const baselineValue = findMetricValue(target.metricId, baselineMetrics);
+  const candidateValue = findMetricValue(target.metricId, trialMetrics);
+  const delta = candidateValue - baselineValue;
+  const minDelta = target.minDelta ?? 0;
+  const passed = isImprovement(delta, target.direction, minDelta);
+
+  return {
+    metricId: target.metricId,
+    baselineValue,
+    candidateValue,
+    delta: normalizeDelta(delta),
+    direction: target.direction,
+    passed,
+  };
+}
+
+function findRegression(
+  target: MetricTarget,
+  baselineMetrics: MetricReading[],
+  trialMetrics: MetricReading[],
+): RegressionFinding[] {
+  const baselineValue = findMetricValue(target.metricId, baselineMetrics);
+  const candidateValue = findMetricValue(target.metricId, trialMetrics);
+  const regressionAmount = calculateRegressionAmount(
+    baselineValue,
+    candidateValue,
+    target.direction,
+  );
+  const limit = target.maxRegression ?? 0;
+
+  if (regressionAmount <= limit) {
+    return [];
+  }
+
+  return [
+    {
+      metricId: target.metricId,
+      baselineValue,
+      candidateValue,
+      regressionAmount: normalizeDelta(regressionAmount),
+      limit,
+    },
+  ];
+}
+
+function isImprovement(delta: number, direction: MetricDirection, minDelta: number): boolean {
+  if (direction === 'increase') {
+    return delta >= minDelta;
+  }
+
+  return -delta >= minDelta;
+}
+
+function calculateRegressionAmount(
+  baselineValue: number,
+  candidateValue: number,
+  direction: MetricDirection,
+): number {
+  if (direction === 'increase') {
+    return Math.max(0, baselineValue - candidateValue);
+  }
+
+  return Math.max(0, candidateValue - baselineValue);
+}
+
+function findMetricValue(metricId: string, metrics: MetricReading[]): number {
+  const metric = metrics.find((reading) => reading.metricId === metricId);
+
+  if (!metric) {
+    return 0;
+  }
+
+  return metric.value;
+}
+
+function isWithinBudget(usage: ResourceUsage, budget: ResourceBudget): boolean {
+  return (
+    usage.durationMs <= budget.maxDurationMs &&
+    usage.computeUnits <= budget.maxComputeUnits &&
+    usage.apiCostUsd <= budget.maxApiCostUsd &&
+    usage.storageBytes <= budget.maxStorageBytes
+  );
+}
+
+function normalizeDelta(value: number): number {
+  return Number(value.toFixed(12));
+}

gitnexus/test/unit/evolver/metric-evaluator.test.ts

@@ -0,0 +1,280 @@
+import { describe, expect, it } from 'vitest';
+import { evaluateVariant } from '../../../src/core/evolver/index.js';
+import type {
+  BaselineProfile,
+  MetricTarget,
+  ResourceBudget,
+  SafetyFinding,
+  TrialResult,
+  VariantSpec,
+} from '../../../src/core/evolver/index.js';
+
+const capturedAt = '2026-05-25T00:00:00.000Z';
+
+function baselineProfile(metricValue = 0.72): BaselineProfile {
+  return {
+    id: 'baseline-1',
+    goalId: 'goal-1',
+    capturedAt,
+    metrics: [
+      {
+        metricId: 'task_success_rate',
+        value: metricValue,
+        capturedAt,
+      },
+    ],
+    resourceUsage: {
+      durationMs: 100,
+      computeUnits: 1,
+      apiCostUsd: 0,
+      storageBytes: 128,
+    },
+  };
+}
+
+function parameterVariant(overrides: Partial<VariantSpec> = {}): VariantSpec {
+  return {
+    id: 'variant-parameter-1',
+    planId: 'plan-1',
+    mutationType: 'parameter',
+    description: 'Increase retrieval topK',
+    patch: {
+      operations: [
+        {
+          op: 'setParameter',
+          path: 'retrieval.topK',
+          previousValue: 5,
+          nextValue: 8,
+        },
+      ],
+    },
+    expectedGain: [
+      {
+        metricId: 'task_success_rate',
+        expectedDelta: 0.05,
+        direction: 'increase',
+      },
+    ],
+    riskLevel: 'low',
+    rollbackPlan: {
+      strategy: 'restore-previous-parameters',
+      operations: [
+        {
+          op: 'setParameter',
+          path: 'retrieval.topK',
+          previousValue: 8,
+          nextValue: 5,
+        },
+      ],
+    },
+    provenance: [
+      {
+        source: 'synthetic-benchmark',
+        reference: 'run-1',
+        capturedAt,
+      },
+    ],
+    ...overrides,
+  };
+}
+
+function trialResult(candidateValue: number, overrides: Partial<TrialResult> = {}): TrialResult {
+  return {
+    id: 'trial-1',
+    variantId: 'variant-parameter-1',
+    protocolId: 'protocol-1',
+    metrics: [
+      {
+        metricId: 'task_success_rate',
+        value: candidateValue,
+        capturedAt,
+      },
+    ],
+    resourceUsage: {
+      durationMs: 120,
+      computeUnits: 1,
+      apiCostUsd: 0,
+      storageBytes: 256,
+    },
+    errors: [],
+    completedAt: capturedAt,
+    ...overrides,
+  };
+}
+
+const targetMetrics: MetricTarget[] = [
+  {
+    metricId: 'task_success_rate',
+    direction: 'increase',
+    minDelta: 0.05,
+    maxRegression: 0.01,
+  },
+];
+
+const budget: ResourceBudget = {
+  maxDurationMs: 1000,
+  maxComputeUnits: 10,
+  maxApiCostUsd: 1,
+  maxStorageBytes: 1024,
+  maxConcurrency: 1,
+  maxHighRiskVariants: 0,
+};
+
+describe('evaluateVariant', () => {
+  it('promotes a parameter variant when metrics improve and checks pass', () => {
+    const report = evaluateVariant({
+      id: 'report-1',
+      variant: parameterVariant(),
+      baseline: baselineProfile(),
+      trial: trialResult(0.81),
+      targetMetrics,
+      budget,
+      safetyFindings: [],
+      rollbackPassed: true,
+    });
+
+    expect(report.verdict).toBe('promote');
+    expect(report.metricDeltas).toEqual([
+      {
+        metricId: 'task_success_rate',
+        baselineValue: 0.72,
+        candidateValue: 0.81,
+        delta: 0.09,
+        direction: 'increase',
+        passed: true,
+      },
+    ]);
+    expect(report.regressions).toEqual([]);
+  });
+
+  it('rejects a variant when a target metric regresses beyond the configured limit', () => {
+    const report = evaluateVariant({
+      id: 'report-2',
+      variant: parameterVariant(),
+      baseline: baselineProfile(),
+      trial: trialResult(0.68),
+      targetMetrics,
+      budget,
+      safetyFindings: [],
+      rollbackPassed: true,
+    });
+
+    expect(report.verdict).toBe('reject');
+    expect(report.metricDeltas[0]?.passed).toBe(false);
+    expect(report.regressions).toEqual([
+      {
+        metricId: 'task_success_rate',
+        baselineValue: 0.72,
+        candidateValue: 0.68,
+        regressionAmount: 0.04,
+        limit: 0.01,
+      },
+    ]);
+  });
+
+  it('rejects a variant when safety findings are present', () => {
+    const safetyFindings: SafetyFinding[] = [
+      {
+        id: 'safety-1',
+        severity: 'high',
+        message: 'unsafe action attempted',
+      },
+    ];
+
+    const report = evaluateVariant({
+      id: 'report-3',
+      variant: parameterVariant(),
+      baseline: baselineProfile(),
+      trial: trialResult(0.81),
+      targetMetrics,
+      budget,
+      safetyFindings,
+      rollbackPassed: true,
+    });
+
+    expect(report.verdict).toBe('reject');
+    expect(report.safetyFindings).toBe(safetyFindings);
+  });
+
+  it('rejects a variant when resource cost exceeds the configured budget', () => {
+    const report = evaluateVariant({
+      id: 'report-budget',
+      variant: parameterVariant(),
+      baseline: baselineProfile(),
+      trial: trialResult(0.81, {
+        resourceUsage: {
+          durationMs: 1200,
+          computeUnits: 1,
+          apiCostUsd: 0,
+          storageBytes: 256,
+        },
+      }),
+      targetMetrics,
+      budget,
+      safetyFindings: [],
+      rollbackPassed: true,
+    });
+
+    expect(report.verdict).toBe('reject');
+    expect(report.resourceCost.durationMs).toBe(1200);
+  });
+
+  it('rejects a variant when rollback validation fails', () => {
+    const report = evaluateVariant({
+      id: 'report-rollback',
+      variant: parameterVariant(),
+      baseline: baselineProfile(),
+      trial: trialResult(0.81),
+      targetMetrics,
+      budget,
+      safetyFindings: [],
+      rollbackPassed: false,
+    });
+
+    expect(report.verdict).toBe('reject');
+    expect(report.metricDeltas[0]?.passed).toBe(true);
+  });
+
+  it('marks an algorithm variant as needs-review even when metrics improve', () => {
+    const algorithmVariant = parameterVariant({
+      id: 'variant-algorithm-1',
+      mutationType: 'algorithm',
+      riskLevel: 'medium',
+      patch: {
+        operations: [
+          {
+            op: 'selectAlgorithm',
+            policyId: 'ranking-policy',
+            previousAlgorithm: 'baseline-ranker',
+            nextAlgorithm: 'candidate-ranker',
+          },
+        ],
+      },
+      rollbackPlan: {
+        strategy: 'disable-candidate',
+        operations: [
+          {
+            op: 'selectAlgorithm',
+            policyId: 'ranking-policy',
+            previousAlgorithm: 'candidate-ranker',
+            nextAlgorithm: 'baseline-ranker',
+          },
+        ],
+      },
+    });
+
+    const report = evaluateVariant({
+      id: 'report-4',
+      variant: algorithmVariant,
+      baseline: baselineProfile(),
+      trial: trialResult(0.83, { variantId: algorithmVariant.id }),
+      targetMetrics,
+      budget,
+      safetyFindings: [],
+      rollbackPassed: true,
+    });
+
+    expect(report.verdict).toBe('needs-review');
+    expect(report.metricDeltas[0]?.passed).toBe(true);
+  });
+});