From 961386af73efe49560b1158edbbbc4e463643728 Mon Sep 17 00:00:00 2001 From: axiomlogicnexus Date: Thu, 7 May 2026 02:58:46 +0200 Subject: [PATCH] Split broader analytics by confidence snapshot --- .../HyperTwistCoachDashboardWidget.cpp | 63 +++++++++++++++- .../HyperTwistTrainingRepositoryLibrary.cpp | 74 +++++++++++++++++++ .../HyperTwistTrainingTypes.h | 12 +++ ...CE_SNAPSHOT_ANALYTICS_PACKET_2026-05-07.md | 54 ++++++++++++++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 docs/arch/HYPERTWIST_PHASE4_TEMPLATE_SIGNAL_CONFIDENCE_SNAPSHOT_ANALYTICS_PACKET_2026-05-07.md diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistCoachDashboardWidget.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistCoachDashboardWidget.cpp index d4cb6d7..3817dbf 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistCoachDashboardWidget.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistCoachDashboardWidget.cpp @@ -117,6 +117,10 @@ namespace HyperTwistCoachDashboardWidgetInternal const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats ); + FString BuildTemplateLaunchAnalyticsTieBreakTemplateSignalConfidenceSnapshotFragment( + const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats + ); + FString BuildTemplateLaunchAnalyticsTieBreakPeerDispositionSplitFragment( const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats ); @@ -180,7 +184,7 @@ namespace HyperTwistCoachDashboardWidgetInternal } return FString::Printf( - TEXT("avg recommended %d | avg focus %d | avg completed attempts %d | total success/failure/timeout/dnf %d/%d/%d/%d | avg mistakes %.2f | review-backed %d | coach-review %d | short set %d | selector split %s | tie-break support %s | peer support split %s | comparable-peer confidence %s | last launch %s | last completed %s"), + TEXT("avg recommended %d | avg focus %d | avg completed attempts %d | total success/failure/timeout/dnf %d/%d/%d/%d | avg mistakes %.2f | review-backed %d | coach-review %d | short set %d | selector split %s | tie-break support %s | template-signal confidence snapshot %s | peer support split %s | comparable-peer confidence %s | last launch %s | last completed %s"), TemplateStats->AverageRecommendedCaseCount, TemplateStats->AverageFocusCaseCount, TemplateStats->AverageCompletedAttemptCount, @@ -194,6 +198,8 @@ namespace HyperTwistCoachDashboardWidgetInternal TemplateStats->ShortReviewSetLaunchCount, *BuildTemplateLaunchSelectionSplitAnalyticsFragment(TemplateStats), *BuildTemplateLaunchAnalyticsTieBreakSupportSplitFragment(TemplateStats), + *BuildTemplateLaunchAnalyticsTieBreakTemplateSignalConfidenceSnapshotFragment( + TemplateStats), *BuildTemplateLaunchAnalyticsTieBreakPeerDispositionSplitFragment(TemplateStats), *BuildTemplateLaunchComparablePeerConfidenceLaneFragment(TemplateStats), TemplateStats->LastLaunchAtUtc.IsEmpty() @@ -525,6 +531,61 @@ namespace HyperTwistCoachDashboardWidgetInternal return Fragment; } + FString BuildTemplateLaunchAnalyticsTieBreakTemplateSignalConfidenceSnapshotFragment( + const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats + ) + { + if (TemplateStats == nullptr || !TemplateStats->IsStructurallyValid()) + { + return TEXT( + "no template-signal confidence snapshot split is available for this template id yet"); + } + + const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& NoRetainedLaneStats = + TemplateStats + ->DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats; + const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& ConfidenceEligibleStats = + TemplateStats + ->DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleStats; + const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& SuppressedStats = + TemplateStats + ->DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyStats; + if (!NoRetainedLaneStats.IsStructurallyValid() + && !ConfidenceEligibleStats.IsStructurallyValid() + && !SuppressedStats.IsStructurallyValid()) + { + return TEXT( + "no template-signal confidence snapshot split is available for this template id yet"); + } + + FString Fragment = FString::Printf( + TEXT("%s | %s | %s"), + *BuildTemplateLaunchSelectionLaneAnalyticsSummary( + ConfidenceEligibleStats, + TEXT("confidence-eligible")), + *BuildTemplateLaunchSelectionLaneAnalyticsSummary( + SuppressedStats, + TEXT("suppressed")), + *BuildTemplateLaunchSelectionLaneAnalyticsSummary( + NoRetainedLaneStats, + TEXT("no-retained-lane")) + ); + if (ConfidenceEligibleStats.IsStructurallyValid() + && SuppressedStats.IsStructurallyValid() + && ConfidenceEligibleStats.GetOutcomeRunCount() > 0 + && SuppressedStats.GetOutcomeRunCount() > 0) + { + Fragment += FString::Printf( + TEXT(" | eligible-minus-suppressed delta success %+.2f | delta mistakes %+.2f"), + ConfidenceEligibleStats.OverallSuccessRate - SuppressedStats.OverallSuccessRate, + ConfidenceEligibleStats.AverageMistakesPerOutcomeRun + - SuppressedStats.AverageMistakesPerOutcomeRun + ); + } + + return Fragment; + } + FString BuildTemplateLaunchAnalyticsTieBreakPeerDispositionSplitFragment( const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats ) diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.cpp index fb557dc..c83442c 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.cpp @@ -9162,6 +9162,12 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser( FSelectionScopedAccumulator DashboardPrimarySelectionAccumulator; FSelectionScopedAccumulator DashboardAnalyticsTieBreakSelectionAccumulator; FSelectionScopedAccumulator DashboardAnalyticsTieBreakTemplateScopedSignalAccumulator; + FSelectionScopedAccumulator + DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator; + FSelectionScopedAccumulator + DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleAccumulator; + FSelectionScopedAccumulator + DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyAccumulator; FSelectionScopedAccumulator DashboardAnalyticsTieBreakSelectorSplitConfidenceAccumulator; FSelectionScopedAccumulator DashboardAnalyticsTieBreakSelectorSplitConfidencePeerLackedComparableEvidenceAccumulator; @@ -9325,6 +9331,38 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser( Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalAccumulator, Provenance.SelectionProvenance ); + const EHyperTwistTrainingTemplateLaunchComparablePeerConfidenceSnapshotState + SnapshotState = + Provenance.SelectionProvenance.ComparablePeerConfidenceSnapshot.State; + if (SnapshotState == + EHyperTwistTrainingTemplateLaunchComparablePeerConfidenceSnapshotState:: + ConfidenceEligible) + { + AccumulateSelectionScopedStats( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleAccumulator, + Provenance.SelectionProvenance + ); + } + else if ( + SnapshotState == + EHyperTwistTrainingTemplateLaunchComparablePeerConfidenceSnapshotState:: + SuppressedPeerNoEvidenceHeavy) + { + AccumulateSelectionScopedStats( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyAccumulator, + Provenance.SelectionProvenance + ); + } + else + { + AccumulateSelectionScopedStats( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator, + Provenance.SelectionProvenance + ); + } } else if (Provenance.SelectionProvenance.AnalyticsSupportKind == EHyperTwistTrainingTemplateLaunchSelectionAnalyticsSupportKind::SelectorSplitConfidence) @@ -9460,6 +9498,18 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser( FinalizeSelectionScopedAccumulator( Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalAccumulator ); + FinalizeSelectionScopedAccumulator( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator + ); + FinalizeSelectionScopedAccumulator( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleAccumulator + ); + FinalizeSelectionScopedAccumulator( + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyAccumulator + ); FinalizeSelectionScopedAccumulator( Accumulator.DashboardAnalyticsTieBreakSelectorSplitConfidenceAccumulator ); @@ -9486,6 +9536,17 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser( Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalAccumulator.Stats .PeerSupportKindDisposition = EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None; + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator + .Stats.PeerSupportKindDisposition = + EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None; + Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleAccumulator + .Stats.PeerSupportKindDisposition = + EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None; + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyAccumulator + .Stats.PeerSupportKindDisposition = + EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None; Accumulator.DashboardAnalyticsTieBreakSelectorSplitConfidenceAccumulator.Stats .PeerSupportKindDisposition = EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None; @@ -9495,6 +9556,19 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser( Accumulator.DashboardAnalyticsTieBreakSelectionAccumulator.Stats; Accumulator.Stats.DashboardAnalyticsTieBreakTemplateScopedSignalStats = Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalAccumulator.Stats; + Accumulator.Stats + .DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats = + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator + .Stats; + Accumulator.Stats.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleStats = + Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleAccumulator + .Stats; + Accumulator.Stats + .DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyStats = + Accumulator + .DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyAccumulator + .Stats; Accumulator.Stats.DashboardAnalyticsTieBreakSelectorSplitConfidenceStats = Accumulator.DashboardAnalyticsTieBreakSelectorSplitConfidenceAccumulator.Stats; Accumulator.Stats diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h index e9fb2ba..33385e6 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h @@ -2245,6 +2245,18 @@ struct FHyperTwistTrainingTemplateLaunchScopedStats FHyperTwistTrainingTemplateLaunchSelectionScopedStats DashboardAnalyticsTieBreakTemplateScopedSignalStats; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistTrainingTemplateLaunchSelectionScopedStats + DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistTrainingTemplateLaunchSelectionScopedStats + DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleStats; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistTrainingTemplateLaunchSelectionScopedStats + DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyStats; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FHyperTwistTrainingTemplateLaunchSelectionScopedStats DashboardAnalyticsTieBreakSelectorSplitConfidenceStats; diff --git a/docs/arch/HYPERTWIST_PHASE4_TEMPLATE_SIGNAL_CONFIDENCE_SNAPSHOT_ANALYTICS_PACKET_2026-05-07.md b/docs/arch/HYPERTWIST_PHASE4_TEMPLATE_SIGNAL_CONFIDENCE_SNAPSHOT_ANALYTICS_PACKET_2026-05-07.md new file mode 100644 index 0000000..6a5cffb --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE4_TEMPLATE_SIGNAL_CONFIDENCE_SNAPSHOT_ANALYTICS_PACKET_2026-05-07.md @@ -0,0 +1,54 @@ +# HyperTwist Phase 4 template-signal confidence snapshot analytics packet + +## Purpose + +This packet pushes the persisted comparable-peer confidence snapshot from per-run provenance into repo-backed template analytics for broader `TemplateScopedOutcomeSignal` analytics tie-break launches. Operators can now compare whether broader analytics picks made while the split-confidence lane was already `confidence-eligible` are outperforming broader picks that launched while that lane was still `suppressed`, without inspecting individual runs one at a time. + +## Scope + +- add repo-backed template-scoped stat buckets for broader analytics launches split by retained confidence snapshot state +- classify broader analytics launches into: + - `confidence-eligible` + - `suppressed` + - `no-retained-lane` +- treat older broader analytics launches without the new persisted snapshot as `no-retained-lane` for continuity +- surface that split through the shared template analytics detail used by the live dashboard, template launch history, and run recap + +## What landed + +- `HyperTwistTrainingTypes.h` + - added three new per-template selection-scoped stat buckets under `FHyperTwistTrainingTemplateLaunchScopedStats` for broader template-signal launches: + - `DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats` + - `DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceEligibleStats` + - `DashboardAnalyticsTieBreakTemplateScopedSignalSuppressedPeerNoEvidenceHeavyStats` +- `HyperTwistTrainingRepositoryLibrary.cpp` + - added matching internal accumulators + - broadened analytics tie-break aggregation so `TemplateScopedOutcomeSignal` launches are routed into the new snapshot-state buckets + - legacy broader analytics launches without the persisted snapshot now accumulate in `no-retained-lane` +- `HyperTwistCoachDashboardWidget.cpp` + - added a shared formatter for the broader template-signal confidence snapshot split + - shared template analytics detail now shows: + - `confidence-eligible` + - `suppressed` + - `no-retained-lane` + - when both eligible and suppressed buckets have outcome history, the detail also reports `eligible-minus-suppressed` delta success and delta mistakes + +## Product effect + +- the live `[Template Launch]` detail now shows whether broader template-signal picks with an already-eligible split-confidence lane are actually beating broader picks launched from a suppressed lane +- template launch history and run recap inherit the same aggregate comparison automatically through the shared analytics fragment +- operators can distinguish between current broader analytics performance that is evidence-backed, still suppressed, or legacy/no-retained-lane + +## Acceptance criteria + +- repo-backed template-scoped stats derive distinct broader template-signal buckets for `confidence-eligible`, `suppressed`, and `no-retained-lane` +- older broader template-signal launches without the persisted snapshot land in `no-retained-lane` +- shared template analytics detail shows the new broader template-signal snapshot split +- when both eligible and suppressed buckets have outcomes, shared detail reports delta success and delta mistakes + +## Validation checklist + +1. full `Development Editor|Win64` build succeeds +2. confirm repository aggregation routes broader template-signal launches into the three new snapshot-state buckets +3. confirm shared analytics detail shows `confidence-eligible`, `suppressed`, and `no-retained-lane` +4. confirm shared detail reports `eligible-minus-suppressed` deltas when both buckets have outcome history