Guard template selector confidence by support kind
This commit is contained in:
parent
c65fc2a312
commit
c41603393b
2 changed files with 115 additions and 15 deletions
|
|
@ -756,8 +756,8 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
}
|
||||
|
||||
constexpr int32 TemplateLaunchAnalyticsTieBreakMinimumOutcomeRuns = 2;
|
||||
constexpr int32 TemplateLaunchSelectionTieBreakConfidenceMinimumTieBreakOutcomeRuns = 3;
|
||||
constexpr int32 TemplateLaunchSelectionTieBreakConfidenceMinimumPrimaryOutcomeRuns = 2;
|
||||
constexpr int32 TemplateLaunchSelectionTieBreakConfidenceMinimumSplitConfidenceOutcomeRuns = 3;
|
||||
constexpr int32 TemplateLaunchSelectionTieBreakConfidenceMinimumTemplateSignalOutcomeRuns = 2;
|
||||
constexpr int32 TemplateLaunchSelectionTieBreakConfidenceMinimumScoreDelta = 2;
|
||||
constexpr float TemplateLaunchSelectionTieBreakConfidenceMinimumSuccessDelta = 0.05f;
|
||||
constexpr float TemplateLaunchSelectionTieBreakConfidenceMinimumMistakeImprovement = 0.5f;
|
||||
|
|
@ -847,29 +847,37 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
return 0;
|
||||
}
|
||||
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& TieBreakStats =
|
||||
TemplateStats->DashboardAnalyticsTieBreakStats;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& PrimaryStats =
|
||||
TemplateStats->DashboardPrimarySelectionStats;
|
||||
const int32 TieBreakOutcomeRuns = CountTemplateLaunchSelectionOutcomeRuns(&TieBreakStats);
|
||||
const int32 PrimaryOutcomeRuns = CountTemplateLaunchSelectionOutcomeRuns(&PrimaryStats);
|
||||
if (TieBreakOutcomeRuns < TemplateLaunchSelectionTieBreakConfidenceMinimumTieBreakOutcomeRuns
|
||||
|| PrimaryOutcomeRuns < TemplateLaunchSelectionTieBreakConfidenceMinimumPrimaryOutcomeRuns)
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& SplitConfidenceStats =
|
||||
TemplateStats->DashboardAnalyticsTieBreakSelectorSplitConfidenceStats;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& TemplateSignalStats =
|
||||
TemplateStats->DashboardAnalyticsTieBreakTemplateScopedSignalStats;
|
||||
const int32 SplitConfidenceOutcomeRuns =
|
||||
CountTemplateLaunchSelectionOutcomeRuns(&SplitConfidenceStats);
|
||||
const int32 TemplateSignalOutcomeRuns =
|
||||
CountTemplateLaunchSelectionOutcomeRuns(&TemplateSignalStats);
|
||||
if (SplitConfidenceOutcomeRuns
|
||||
< TemplateLaunchSelectionTieBreakConfidenceMinimumSplitConfidenceOutcomeRuns
|
||||
|| TemplateSignalOutcomeRuns
|
||||
< TemplateLaunchSelectionTieBreakConfidenceMinimumTemplateSignalOutcomeRuns)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
const int32 TieBreakScore = BuildTemplateLaunchSelectionLaneAnalyticsScore(&TieBreakStats);
|
||||
const int32 PrimaryScore = BuildTemplateLaunchSelectionLaneAnalyticsScore(&PrimaryStats);
|
||||
const int32 ScoreDelta = TieBreakScore - PrimaryScore;
|
||||
const int32 SplitConfidenceScore =
|
||||
BuildTemplateLaunchSelectionLaneAnalyticsScore(&SplitConfidenceStats);
|
||||
const int32 TemplateSignalScore =
|
||||
BuildTemplateLaunchSelectionLaneAnalyticsScore(&TemplateSignalStats);
|
||||
const int32 ScoreDelta = SplitConfidenceScore - TemplateSignalScore;
|
||||
if (ScoreDelta <= 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
const float SuccessDelta = TieBreakStats.OverallSuccessRate - PrimaryStats.OverallSuccessRate;
|
||||
const float SuccessDelta =
|
||||
SplitConfidenceStats.OverallSuccessRate - TemplateSignalStats.OverallSuccessRate;
|
||||
const float MistakeImprovement =
|
||||
PrimaryStats.AverageMistakesPerOutcomeRun - TieBreakStats.AverageMistakesPerOutcomeRun;
|
||||
TemplateSignalStats.AverageMistakesPerOutcomeRun
|
||||
- SplitConfidenceStats.AverageMistakesPerOutcomeRun;
|
||||
if (ScoreDelta < TemplateLaunchSelectionTieBreakConfidenceMinimumScoreDelta
|
||||
&& SuccessDelta < TemplateLaunchSelectionTieBreakConfidenceMinimumSuccessDelta
|
||||
&& MistakeImprovement < TemplateLaunchSelectionTieBreakConfidenceMinimumMistakeImprovement)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
# HyperTwist Phase 4 template launch support-kind confidence guard packet
|
||||
|
||||
Created on `2026-05-07`
|
||||
|
||||
Status:
|
||||
|
||||
- first-party HyperTwist packet
|
||||
- bounded Phase `4` dashboard selector refinement slice
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet replaces the selector-split confidence bonus guard so it is backed by repo-derived support-kind buckets instead of the broader analytics-tie-break lane as a whole.
|
||||
|
||||
The open tasks were:
|
||||
|
||||
- stop inferring selector-split confidence from mixed analytics tie-break history
|
||||
- require the stored `SelectorSplitConfidence` bucket itself to have enough outcome history before it can strengthen a future tied selection
|
||||
- compare that bucket directly against the template's `TemplateScopedOutcomeSignal` tie-break bucket rather than against non-tie-break dashboard priority history
|
||||
|
||||
It is not:
|
||||
|
||||
- a new repository aggregation packet
|
||||
- another analytics provenance packet
|
||||
- a change to primary dashboard template ranking rules
|
||||
- a new dashboard surface packet
|
||||
|
||||
## Scope
|
||||
|
||||
Bounded lane:
|
||||
|
||||
- retarget the dashboard selector confidence guard to repo-backed support-kind buckets already present on template launch scoped stats
|
||||
- require enough stored outcome runs on the `SelectorSplitConfidence` bucket and the `TemplateScopedOutcomeSignal` bucket before awarding the extra confidence point
|
||||
- keep the same small confidence bonus and the same bounded score/success/mistake thresholds
|
||||
- leave the rest of the overall analytics tie-break path unchanged
|
||||
|
||||
Out of scope:
|
||||
|
||||
- changing support-kind aggregation itself
|
||||
- changing the existing template-scoped analytics tie-break score
|
||||
- changing selection provenance payloads
|
||||
- adding a new dashboard explanation surface
|
||||
|
||||
## Why this was the right next packet
|
||||
|
||||
The previous packet made support-kind split outcomes visible in repo-backed template analytics.
|
||||
|
||||
That immediately exposed a selector gap:
|
||||
|
||||
- the dashboard could now show support-kind split performance
|
||||
- but the selector confidence bonus still looked at the broader analytics tie-break lane instead of the support-kind split itself
|
||||
- which meant mixed tie-break history could still award selector-split confidence even if the explicit `SelectorSplitConfidence` bucket had not independently proven itself
|
||||
|
||||
So the next honest move was:
|
||||
|
||||
- keep the bonus small
|
||||
- keep it attached only to tied-template analytics selection
|
||||
- but make it answer to the actual support-kind split that now exists in stored analytics
|
||||
|
||||
## What landed
|
||||
|
||||
Primary code changes:
|
||||
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistCoachDashboardWidget.cpp`
|
||||
- retargeted `BuildTemplateLaunchAnalyticsTieBreakConfidenceBonus(...)` to compare `DashboardAnalyticsTieBreakSelectorSplitConfidenceStats` against `DashboardAnalyticsTieBreakTemplateScopedSignalStats`
|
||||
- requires enough stored outcome runs on both support-kind buckets before awarding confidence
|
||||
- preserves the existing bounded score delta, success delta, and mistake-improvement thresholds
|
||||
- leaves the broader equal-priority analytics tie-break path unchanged outside this confidence source swap
|
||||
|
||||
## Product effect
|
||||
|
||||
Dashboard selector-split confidence is now grounded in the right evidence:
|
||||
|
||||
- a template only earns the extra selector-split confidence point when its explicit `SelectorSplitConfidence` history has enough outcomes
|
||||
- that bucket must outperform the template's own `TemplateScopedOutcomeSignal` tie-break history, not merely the broader mixed analytics lane
|
||||
- sparse or mixed tie-break history no longer overstates selector-split confidence
|
||||
- tied-template selection remains conservative and repo-backed
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- selector-split confidence bonus uses repo-backed support-kind buckets
|
||||
- bonus only applies when the `SelectorSplitConfidence` bucket has enough outcome history
|
||||
- bonus only applies when the `SelectorSplitConfidence` bucket outperforms the `TemplateScopedOutcomeSignal` bucket on meaningful stored signal
|
||||
- full product build succeeds
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. build `UnrealHyperTwist.sln` / `UnrealHyperTwistEditor`
|
||||
2. confirm selector confidence bonus now reads support-kind split stats instead of the broader analytics tie-break lane
|
||||
3. confirm sparse or underperforming `SelectorSplitConfidence` history does not award the extra confidence point
|
||||
4. confirm the rest of the tied-template analytics selection path remains unchanged
|
||||
|
||||
That is the packet.
|
||||
Loading…
Add table
Reference in a new issue