Guard template decision bias by snapshot history
This commit is contained in:
axiomlogicnexus
parent
2fdcaf46ee
commit
8bf4e0975c
2 changed files with 249 additions and 17 deletions
|
|
@ -220,6 +220,12 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
int32* OutPlainSnapshotGuardOutcomeRuns = nullptr
|
||||
);
|
||||
|
||||
bool HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats,
|
||||
int32* OutEarnedOutcomeRuns = nullptr,
|
||||
int32* OutComparisonOutcomeRuns = nullptr
|
||||
);
|
||||
|
||||
int32 BuildTemplateLaunchAnalyticsTieBreakConfidenceBonus(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
);
|
||||
|
|
@ -1030,6 +1036,20 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
- SuppressedStats.AverageMistakesPerOutcomeRun
|
||||
);
|
||||
}
|
||||
int32 EarnedSelectionBiasOutcomeRuns = 0;
|
||||
int32 ComparisonSelectionBiasOutcomeRuns = 0;
|
||||
const bool bSelectionBiasEarned =
|
||||
HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(
|
||||
TemplateStats,
|
||||
&EarnedSelectionBiasOutcomeRuns,
|
||||
&ComparisonSelectionBiasOutcomeRuns
|
||||
);
|
||||
Fragment += FString::Printf(
|
||||
TEXT(" | selection-bias %s earned outcomes %d | comparison outcomes %d"),
|
||||
bSelectionBiasEarned ? TEXT("earned") : TEXT("suppressed"),
|
||||
EarnedSelectionBiasOutcomeRuns,
|
||||
ComparisonSelectionBiasOutcomeRuns
|
||||
);
|
||||
|
||||
return Fragment;
|
||||
}
|
||||
|
|
@ -2018,6 +2038,18 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
constexpr int32 TemplateLaunchTemplateSignalSupportDecisionMinimumScoreDelta = 1;
|
||||
constexpr float TemplateLaunchTemplateSignalSupportDecisionMinimumSuccessDelta = 0.05f;
|
||||
constexpr float TemplateLaunchTemplateSignalSupportDecisionMinimumMistakeImprovement = 0.5f;
|
||||
constexpr int32
|
||||
TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumEarnedOutcomeRuns = 2;
|
||||
constexpr int32
|
||||
TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumComparisonOutcomeRuns =
|
||||
2;
|
||||
constexpr int32 TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumScoreDelta =
|
||||
1;
|
||||
constexpr float
|
||||
TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumSuccessDelta = 0.05f;
|
||||
constexpr float
|
||||
TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumMistakeImprovement =
|
||||
0.5f;
|
||||
|
||||
int32 CountTemplateLaunchOutcomeRuns(const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats)
|
||||
{
|
||||
|
|
@ -2491,6 +2523,111 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
>= TemplateLaunchTemplateSignalSupportDecisionMinimumMistakeImprovement;
|
||||
}
|
||||
|
||||
bool HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats,
|
||||
int32* OutEarnedOutcomeRuns,
|
||||
int32* OutComparisonOutcomeRuns
|
||||
)
|
||||
{
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats* EarnedStats = nullptr;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats* SuppressedStats = nullptr;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats* NoRetainedStats = nullptr;
|
||||
if (TemplateStats != nullptr && TemplateStats->IsStructurallyValid())
|
||||
{
|
||||
EarnedStats =
|
||||
&TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportDecisionBiasSnapshotEarnedStats;
|
||||
SuppressedStats =
|
||||
&TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportDecisionBiasSnapshotSuppressedStats;
|
||||
NoRetainedStats =
|
||||
&TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportDecisionBiasSnapshotNoRetainedStats;
|
||||
}
|
||||
|
||||
const int32 EarnedOutcomeRuns = CountTemplateLaunchSelectionOutcomeRuns(EarnedStats);
|
||||
const int32 SuppressedOutcomeRuns = CountTemplateLaunchSelectionOutcomeRuns(SuppressedStats);
|
||||
const int32 NoRetainedOutcomeRuns = CountTemplateLaunchSelectionOutcomeRuns(NoRetainedStats);
|
||||
const int32 ComparisonOutcomeRuns = SuppressedOutcomeRuns + NoRetainedOutcomeRuns;
|
||||
if (OutEarnedOutcomeRuns != nullptr)
|
||||
{
|
||||
*OutEarnedOutcomeRuns = EarnedOutcomeRuns;
|
||||
}
|
||||
if (OutComparisonOutcomeRuns != nullptr)
|
||||
{
|
||||
*OutComparisonOutcomeRuns = ComparisonOutcomeRuns;
|
||||
}
|
||||
|
||||
if (!HasTemplateLaunchTemplateSignalSupportDecisionBias(TemplateStats)
|
||||
|| EarnedOutcomeRuns
|
||||
< TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumEarnedOutcomeRuns
|
||||
|| ComparisonOutcomeRuns
|
||||
< TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumComparisonOutcomeRuns)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
const int32 ComparisonCompletedAttemptCount =
|
||||
(SuppressedStats != nullptr && SuppressedStats->IsStructurallyValid()
|
||||
? SuppressedStats->TotalCompletedAttemptCount
|
||||
: 0)
|
||||
+ (NoRetainedStats != nullptr && NoRetainedStats->IsStructurallyValid()
|
||||
? NoRetainedStats->TotalCompletedAttemptCount
|
||||
: 0);
|
||||
const int32 ComparisonSuccessCount =
|
||||
(SuppressedStats != nullptr && SuppressedStats->IsStructurallyValid()
|
||||
? SuppressedStats->TotalSuccessCount
|
||||
: 0)
|
||||
+ (NoRetainedStats != nullptr && NoRetainedStats->IsStructurallyValid()
|
||||
? NoRetainedStats->TotalSuccessCount
|
||||
: 0);
|
||||
const int32 ComparisonAbortedRunCount =
|
||||
(SuppressedStats != nullptr && SuppressedStats->IsStructurallyValid()
|
||||
? SuppressedStats->AbortedRunCount
|
||||
: 0)
|
||||
+ (NoRetainedStats != nullptr && NoRetainedStats->IsStructurallyValid()
|
||||
? NoRetainedStats->AbortedRunCount
|
||||
: 0);
|
||||
const float ComparisonSuccessRate = ComparisonCompletedAttemptCount > 0
|
||||
? static_cast<float>(ComparisonSuccessCount)
|
||||
/ static_cast<float>(ComparisonCompletedAttemptCount)
|
||||
: 0.0f;
|
||||
const float ComparisonAverageMistakesPerOutcomeRun = ComparisonOutcomeRuns > 0
|
||||
? (((SuppressedStats != nullptr && SuppressedStats->IsStructurallyValid()
|
||||
? SuppressedStats->AverageMistakesPerOutcomeRun
|
||||
: 0.0f)
|
||||
* static_cast<float>(SuppressedOutcomeRuns))
|
||||
+ ((NoRetainedStats != nullptr && NoRetainedStats->IsStructurallyValid()
|
||||
? NoRetainedStats->AverageMistakesPerOutcomeRun
|
||||
: 0.0f)
|
||||
* static_cast<float>(NoRetainedOutcomeRuns)))
|
||||
/ static_cast<float>(ComparisonOutcomeRuns)
|
||||
: 0.0f;
|
||||
const int32 EarnedScore = BuildTemplateLaunchSelectionLaneAnalyticsScore(EarnedStats);
|
||||
const int32 ComparisonScore = BuildTemplateLaunchAnalyticsOutcomeSignalScore(
|
||||
ComparisonOutcomeRuns,
|
||||
ComparisonSuccessRate,
|
||||
ComparisonAverageMistakesPerOutcomeRun,
|
||||
ComparisonAbortedRunCount
|
||||
);
|
||||
const int32 ScoreDelta = EarnedScore - ComparisonScore;
|
||||
const float SuccessDelta =
|
||||
(EarnedStats != nullptr && EarnedStats->IsStructurallyValid()
|
||||
? EarnedStats->OverallSuccessRate
|
||||
: 0.0f)
|
||||
- ComparisonSuccessRate;
|
||||
const float MistakeImprovement = ComparisonAverageMistakesPerOutcomeRun
|
||||
- (EarnedStats != nullptr && EarnedStats->IsStructurallyValid()
|
||||
? EarnedStats->AverageMistakesPerOutcomeRun
|
||||
: 0.0f);
|
||||
return ScoreDelta
|
||||
>= TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumScoreDelta
|
||||
|| SuccessDelta
|
||||
>= TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumSuccessDelta
|
||||
|| MistakeImprovement
|
||||
>= TemplateLaunchTemplateSignalSupportDecisionBiasSnapshotMinimumMistakeImprovement;
|
||||
}
|
||||
|
||||
int32 BuildTemplateLaunchAnalyticsTieBreakConfidenceBonus(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
)
|
||||
|
|
@ -2760,31 +2897,72 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
&& HasTemplateLaunchTemplateSignalSupportBiasSnapshotBias(IncumbentStats)
|
||||
? 1
|
||||
: 0;
|
||||
const int32 CandidateTemplateSignalSupportBiasBonus =
|
||||
CandidateTemplateSignalSupportBiasSnapshotGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBias(CandidateStats)
|
||||
? 1
|
||||
: 0;
|
||||
const int32 IncumbentTemplateSignalSupportBiasBonus =
|
||||
IncumbentTemplateSignalSupportBiasSnapshotGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBias(IncumbentStats)
|
||||
? 1
|
||||
: 0;
|
||||
if (CandidateTemplateSignalSupportBiasBonus
|
||||
!= IncumbentTemplateSignalSupportBiasBonus)
|
||||
if (CandidateTemplateSignalSupportBiasSnapshotGuardBonus
|
||||
!= IncumbentTemplateSignalSupportBiasSnapshotGuardBonus)
|
||||
{
|
||||
Decision.bPreferCandidate =
|
||||
CandidateTemplateSignalSupportBiasBonus
|
||||
> IncumbentTemplateSignalSupportBiasBonus;
|
||||
CandidateTemplateSignalSupportBiasSnapshotGuardBonus
|
||||
> IncumbentTemplateSignalSupportBiasSnapshotGuardBonus;
|
||||
if (Decision.bPreferCandidate)
|
||||
{
|
||||
Decision.CandidateAnalyticsSupportKind =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionAnalyticsSupportKind::
|
||||
TemplateScopedOutcomeSignal;
|
||||
Decision.bCandidateUsedTemplateSignalSupportBiasSnapshotGuard = true;
|
||||
Decision.bCandidateUsedTemplateSignalSupportDecisionBiasGuard =
|
||||
CandidateTemplateSignalSupportBiasSnapshotGuardBonus
|
||||
== IncumbentTemplateSignalSupportBiasSnapshotGuardBonus;
|
||||
}
|
||||
return Decision;
|
||||
}
|
||||
const int32 CandidateTemplateSignalSupportDecisionGuardBonus =
|
||||
CandidateTemplateSignalSupportBiasSnapshotGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBias(CandidateStats)
|
||||
? 1
|
||||
: 0;
|
||||
const int32 IncumbentTemplateSignalSupportDecisionGuardBonus =
|
||||
IncumbentTemplateSignalSupportBiasSnapshotGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBias(IncumbentStats)
|
||||
? 1
|
||||
: 0;
|
||||
if (CandidateTemplateSignalSupportDecisionGuardBonus
|
||||
!= IncumbentTemplateSignalSupportDecisionGuardBonus)
|
||||
{
|
||||
Decision.bPreferCandidate =
|
||||
CandidateTemplateSignalSupportDecisionGuardBonus
|
||||
> IncumbentTemplateSignalSupportDecisionGuardBonus;
|
||||
if (Decision.bPreferCandidate)
|
||||
{
|
||||
Decision.CandidateAnalyticsSupportKind =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionAnalyticsSupportKind::
|
||||
TemplateScopedOutcomeSignal;
|
||||
Decision.bCandidateUsedTemplateSignalSupportBiasSnapshotGuard = true;
|
||||
Decision.bCandidateUsedTemplateSignalSupportDecisionBiasGuard = true;
|
||||
}
|
||||
return Decision;
|
||||
}
|
||||
const int32 CandidateTemplateSignalSupportDecisionBiasSnapshotGuardBonus =
|
||||
CandidateTemplateSignalSupportDecisionGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(
|
||||
CandidateStats)
|
||||
? 1
|
||||
: 0;
|
||||
const int32 IncumbentTemplateSignalSupportDecisionBiasSnapshotGuardBonus =
|
||||
IncumbentTemplateSignalSupportDecisionGuardBonus > 0
|
||||
&& HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(
|
||||
IncumbentStats)
|
||||
? 1
|
||||
: 0;
|
||||
if (CandidateTemplateSignalSupportDecisionBiasSnapshotGuardBonus
|
||||
!= IncumbentTemplateSignalSupportDecisionBiasSnapshotGuardBonus)
|
||||
{
|
||||
Decision.bPreferCandidate =
|
||||
CandidateTemplateSignalSupportDecisionBiasSnapshotGuardBonus
|
||||
> IncumbentTemplateSignalSupportDecisionBiasSnapshotGuardBonus;
|
||||
if (Decision.bPreferCandidate)
|
||||
{
|
||||
Decision.CandidateAnalyticsSupportKind =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionAnalyticsSupportKind::
|
||||
TemplateScopedOutcomeSignal;
|
||||
Decision.bCandidateUsedTemplateSignalSupportBiasSnapshotGuard = true;
|
||||
Decision.bCandidateUsedTemplateSignalSupportDecisionBiasGuard = true;
|
||||
}
|
||||
return Decision;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
# HyperTwist Phase 4 template-signal decision-bias snapshot selection guard packet
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet feeds the new at-launch `earned` versus `suppressed/no-retained` decision-bias snapshot split back into tied broader template-signal analytics selection. A template no longer gets the extra broader template-signal preference from the stricter `DecisionSplitGuard` lane just because its current reinforced-vs-plain snapshot-guard lane looks healthy; it also needs stored launches made under an already-earned decision-bias snapshot to beat its suppressed or legacy no-retained history.
|
||||
|
||||
## Scope
|
||||
|
||||
- add a historical decision-bias snapshot guard helper in dashboard selection logic
|
||||
- require the existing current `HasTemplateLaunchTemplateSignalSupportDecisionBias(...)` guard to clear first
|
||||
- require minimum outcome history in both:
|
||||
- the at-launch `earned` decision-bias snapshot bucket
|
||||
- the combined non-earned comparison bucket (`suppressed + no-retained`)
|
||||
- require the earned snapshot bucket to beat that combined non-earned bucket on score or meaningful success / mistake improvement
|
||||
- use that stricter combined guard only after templates are already tied through the existing decision-split lane
|
||||
- surface the same `selection-bias earned/suppressed` state in the shared decision-bias snapshot analytics fragment
|
||||
|
||||
## What landed
|
||||
|
||||
- `HyperTwistCoachDashboardWidget.cpp`
|
||||
- added `HasTemplateLaunchTemplateSignalSupportDecisionBiasSnapshotBias(...)`
|
||||
- the helper:
|
||||
- reuses the existing current decision-bias guard as a prerequisite
|
||||
- requires at least `2` earned snapshot outcome runs
|
||||
- requires at least `2` comparison outcome runs across `suppressed + no-retained`
|
||||
- compares the earned snapshot lane against the combined non-earned baseline on score, success, and mistakes
|
||||
- `ResolveTemplateAnalyticsPreferenceDecision(...)` now tiers broader template-signal tie-breaks as:
|
||||
- support-bias snapshot guard
|
||||
- current decision-bias guard
|
||||
- historical decision-bias snapshot guard
|
||||
- shared `template-signal decision-bias snapshot` analytics detail now appends:
|
||||
- `selection-bias earned`
|
||||
- or `selection-bias suppressed`
|
||||
- plus earned and comparison outcome counts
|
||||
|
||||
## Product effect
|
||||
|
||||
- the stricter `DecisionSplitGuard` lane now only reinforces future tied broader selections when stored launches made under an already-earned decision-bias snapshot are outperforming suppressed or legacy no-retained history
|
||||
- sparse or weaker earned snapshot history no longer gets to reinforce that stricter lane by itself
|
||||
- live `[Template Launch]` detail, template history, and recap now show whether the historical decision-bias snapshot guard is currently earned or suppressed
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- tied broader-analytics selection uses the new historical decision-bias snapshot guard only after the current decision-bias lane already ties
|
||||
- the new guard requires minimum earned and comparison outcome counts
|
||||
- the new guard only clears when earned decision-bias snapshot launches beat combined suppressed/no-retained history on score or meaningful quality deltas
|
||||
- shared analytics detail shows the historical `selection-bias earned/suppressed` state for the decision-bias snapshot split
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. full `Development Editor|Win64` build succeeds
|
||||
2. confirm `ResolveTemplateAnalyticsPreferenceDecision(...)` tiers broader template-signal tie-breaks through support-bias snapshot, current decision-bias, then historical decision-bias snapshot
|
||||
3. confirm the historical guard compares earned decision-bias snapshot history against combined suppressed/no-retained history with minimum-sample thresholds
|
||||
4. confirm shared analytics detail reports `selection-bias earned` or `selection-bias suppressed` on the decision-bias snapshot split
|
||||
Loading…
Add table
Reference in a new issue