Split template analytics by support-bias snapshot
This commit is contained in:
axiomlogicnexus
parent
a46404dda6
commit
01438a4e15
4 changed files with 199 additions and 2 deletions
|
|
@ -121,6 +121,10 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
);
|
||||
|
||||
FString BuildTemplateLaunchAnalyticsTieBreakTemplateSignalSupportBiasSnapshotFragment(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
);
|
||||
|
||||
FString BuildTemplateLaunchAnalyticsTieBreakTemplateSignalConfidenceSnapshotFragment(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
);
|
||||
|
|
@ -211,7 +215,7 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
}
|
||||
|
||||
return FString::Printf(
|
||||
TEXT("avg recommended %d | avg focus %d | avg completed attempts %d | total success/failure/timeout/dnf %d/%d/%d/%d | avg mistakes %.2f | review-backed %d | coach-review %d | short set %d | selector split %s | tie-break support %s | template-signal support %s | template-signal confidence snapshot %s | peer support split %s | comparable-peer confidence %s | last launch %s | last completed %s"),
|
||||
TEXT("avg recommended %d | avg focus %d | avg completed attempts %d | total success/failure/timeout/dnf %d/%d/%d/%d | avg mistakes %.2f | review-backed %d | coach-review %d | short set %d | selector split %s | tie-break support %s | template-signal support %s | template-signal support-bias snapshot %s | template-signal confidence snapshot %s | peer support split %s | comparable-peer confidence %s | last launch %s | last completed %s"),
|
||||
TemplateStats->AverageRecommendedCaseCount,
|
||||
TemplateStats->AverageFocusCaseCount,
|
||||
TemplateStats->AverageCompletedAttemptCount,
|
||||
|
|
@ -226,6 +230,8 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
*BuildTemplateLaunchSelectionSplitAnalyticsFragment(TemplateStats),
|
||||
*BuildTemplateLaunchAnalyticsTieBreakSupportSplitFragment(TemplateStats),
|
||||
*BuildTemplateLaunchAnalyticsTieBreakTemplateSignalSupportSplitFragment(TemplateStats),
|
||||
*BuildTemplateLaunchAnalyticsTieBreakTemplateSignalSupportBiasSnapshotFragment(
|
||||
TemplateStats),
|
||||
*BuildTemplateLaunchAnalyticsTieBreakTemplateSignalConfidenceSnapshotFragment(
|
||||
TemplateStats),
|
||||
*BuildTemplateLaunchAnalyticsTieBreakPeerDispositionSplitFragment(TemplateStats),
|
||||
|
|
@ -723,7 +729,7 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
&BroaderOutcomeOnlyOutcomeRuns
|
||||
);
|
||||
Fragment += FString::Printf(
|
||||
TEXT(" | support-bias %s bias outcomes %d | broader outcomes %d"),
|
||||
TEXT(" | current support-bias %s bias outcomes %d | broader outcomes %d"),
|
||||
bSupportBiasEarned ? TEXT("earned") : TEXT("suppressed"),
|
||||
ConfidenceLaneBiasOutcomeRuns,
|
||||
BroaderOutcomeOnlyOutcomeRuns
|
||||
|
|
@ -732,6 +738,57 @@ namespace HyperTwistCoachDashboardWidgetInternal
|
|||
return Fragment;
|
||||
}
|
||||
|
||||
FString BuildTemplateLaunchAnalyticsTieBreakTemplateSignalSupportBiasSnapshotFragment(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
)
|
||||
{
|
||||
if (TemplateStats == nullptr || !TemplateStats->IsStructurallyValid())
|
||||
{
|
||||
return TEXT(
|
||||
"no template-signal support-bias snapshot split is available for this template id yet");
|
||||
}
|
||||
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& EarnedStats =
|
||||
TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedStats;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& SuppressedStats =
|
||||
TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedStats;
|
||||
const FHyperTwistTrainingTemplateLaunchSelectionScopedStats& NoRetainedStats =
|
||||
TemplateStats
|
||||
->DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedStats;
|
||||
if (!EarnedStats.IsStructurallyValid()
|
||||
&& !SuppressedStats.IsStructurallyValid()
|
||||
&& !NoRetainedStats.IsStructurallyValid())
|
||||
{
|
||||
return TEXT(
|
||||
"no template-signal support-bias snapshot split is available for this template id yet");
|
||||
}
|
||||
|
||||
FString Fragment = FString::Printf(
|
||||
TEXT("%s | %s | %s"),
|
||||
*BuildTemplateLaunchSelectionLaneAnalyticsSummary(EarnedStats, TEXT("earned")),
|
||||
*BuildTemplateLaunchSelectionLaneAnalyticsSummary(SuppressedStats, TEXT("suppressed")),
|
||||
*BuildTemplateLaunchSelectionLaneAnalyticsSummary(
|
||||
NoRetainedStats,
|
||||
TEXT("no-retained"))
|
||||
);
|
||||
if (EarnedStats.IsStructurallyValid()
|
||||
&& SuppressedStats.IsStructurallyValid()
|
||||
&& EarnedStats.GetOutcomeRunCount() > 0
|
||||
&& SuppressedStats.GetOutcomeRunCount() > 0)
|
||||
{
|
||||
Fragment += FString::Printf(
|
||||
TEXT(" | earned-minus-suppressed delta success %+.2f | delta mistakes %+.2f"),
|
||||
EarnedStats.OverallSuccessRate - SuppressedStats.OverallSuccessRate,
|
||||
EarnedStats.AverageMistakesPerOutcomeRun
|
||||
- SuppressedStats.AverageMistakesPerOutcomeRun
|
||||
);
|
||||
}
|
||||
|
||||
return Fragment;
|
||||
}
|
||||
|
||||
FString BuildTemplateLaunchAnalyticsTieBreakPeerDispositionSplitFragment(
|
||||
const FHyperTwistTrainingTemplateLaunchScopedStats* TemplateStats
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9166,6 +9166,12 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser(
|
|||
DashboardAnalyticsTieBreakTemplateScopedSignalBroaderOutcomeOnlyAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceLaneBiasAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator;
|
||||
FSelectionScopedAccumulator
|
||||
|
|
@ -9353,6 +9359,36 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser(
|
|||
Provenance.SelectionProvenance
|
||||
);
|
||||
}
|
||||
const EHyperTwistTrainingTemplateLaunchTemplateSignalSupportBiasSnapshotState
|
||||
SupportBiasSnapshotState =
|
||||
Provenance.SelectionProvenance.TemplateSignalSupportBiasSnapshot.State;
|
||||
if (SupportBiasSnapshotState ==
|
||||
EHyperTwistTrainingTemplateLaunchTemplateSignalSupportBiasSnapshotState::Earned)
|
||||
{
|
||||
AccumulateSelectionScopedStats(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedAccumulator,
|
||||
Provenance.SelectionProvenance
|
||||
);
|
||||
}
|
||||
else if (
|
||||
SupportBiasSnapshotState ==
|
||||
EHyperTwistTrainingTemplateLaunchTemplateSignalSupportBiasSnapshotState::Suppressed)
|
||||
{
|
||||
AccumulateSelectionScopedStats(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedAccumulator,
|
||||
Provenance.SelectionProvenance
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
AccumulateSelectionScopedStats(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedAccumulator,
|
||||
Provenance.SelectionProvenance
|
||||
);
|
||||
}
|
||||
const EHyperTwistTrainingTemplateLaunchComparablePeerConfidenceSnapshotState
|
||||
SnapshotState =
|
||||
Provenance.SelectionProvenance.ComparablePeerConfidenceSnapshot.State;
|
||||
|
|
@ -9528,6 +9564,18 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser(
|
|||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceLaneBiasAccumulator
|
||||
);
|
||||
FinalizeSelectionScopedAccumulator(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedAccumulator
|
||||
);
|
||||
FinalizeSelectionScopedAccumulator(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedAccumulator
|
||||
);
|
||||
FinalizeSelectionScopedAccumulator(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedAccumulator
|
||||
);
|
||||
FinalizeSelectionScopedAccumulator(
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator
|
||||
|
|
@ -9572,6 +9620,18 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser(
|
|||
Accumulator.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceLaneBiasAccumulator
|
||||
.Stats.PeerSupportKindDisposition =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None;
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedAccumulator
|
||||
.Stats.PeerSupportKindDisposition =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None;
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedAccumulator
|
||||
.Stats.PeerSupportKindDisposition =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None;
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedAccumulator
|
||||
.Stats.PeerSupportKindDisposition =
|
||||
EHyperTwistTrainingTemplateLaunchSelectionPeerSupportKindDisposition::None;
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneAccumulator
|
||||
.Stats.PeerSupportKindDisposition =
|
||||
|
|
@ -9600,6 +9660,21 @@ UHyperTwistTrainingRepositoryLibrary::ListTemplateLaunchScopedStatsForUser(
|
|||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceLaneBiasAccumulator
|
||||
.Stats;
|
||||
Accumulator.Stats
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedStats =
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedAccumulator
|
||||
.Stats;
|
||||
Accumulator.Stats
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedStats =
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedAccumulator
|
||||
.Stats;
|
||||
Accumulator.Stats
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedStats =
|
||||
Accumulator
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedAccumulator
|
||||
.Stats;
|
||||
Accumulator.Stats
|
||||
.DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats =
|
||||
Accumulator
|
||||
|
|
|
|||
|
|
@ -2299,6 +2299,18 @@ struct FHyperTwistTrainingTemplateLaunchScopedStats
|
|||
FHyperTwistTrainingTemplateLaunchSelectionScopedStats
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalConfidenceLaneBiasStats;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistTrainingTemplateLaunchSelectionScopedStats
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedStats;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistTrainingTemplateLaunchSelectionScopedStats
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedStats;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistTrainingTemplateLaunchSelectionScopedStats
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedStats;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistTrainingTemplateLaunchSelectionScopedStats
|
||||
DashboardAnalyticsTieBreakTemplateScopedSignalNoRetainedComparablePeerConfidenceLaneStats;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
# HyperTwist Phase 4 template-signal support-bias snapshot analytics packet
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet pushes the new at-launch template-signal `support-bias earned/suppressed` snapshot back into repo-backed template analytics. The dashboard can now compare broader `TemplateScopedOutcomeSignal` outcomes that launched while support-bias was already earned against outcomes that launched while that lane was still suppressed, instead of only showing the current guard state.
|
||||
|
||||
## Scope
|
||||
|
||||
- add repo-backed broader template-signal snapshot buckets for:
|
||||
- `earned`
|
||||
- `suppressed`
|
||||
- `no-retained` for older launches before the new persisted snapshot existed
|
||||
- route broader `TemplateScopedOutcomeSignal` launches into those buckets from stored selection provenance
|
||||
- surface the split through shared template analytics detail used by live detail, template history, and recap
|
||||
- report an `earned-minus-suppressed` success and mistake delta when both snapshot lanes have outcome history
|
||||
- rename the existing live support split clause to `current support-bias` so it is not confused with the new at-launch snapshot analytics
|
||||
|
||||
## What landed
|
||||
|
||||
- `HyperTwistTrainingTypes.h`
|
||||
- added:
|
||||
- `DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotEarnedStats`
|
||||
- `DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotSuppressedStats`
|
||||
- `DashboardAnalyticsTieBreakTemplateScopedSignalSupportBiasSnapshotNoRetainedStats`
|
||||
- `HyperTwistTrainingRepositoryLibrary.cpp`
|
||||
- widened the repo accumulator with matching support-bias snapshot buckets
|
||||
- broader template-signal launches now route by stored `TemplateSignalSupportBiasSnapshot.State`
|
||||
- older broader launches with no retained snapshot flow into `no-retained`
|
||||
- finalizes and persists all three snapshot buckets into repo-backed template analytics
|
||||
- `HyperTwistCoachDashboardWidget.cpp`
|
||||
- added `BuildTemplateLaunchAnalyticsTieBreakTemplateSignalSupportBiasSnapshotFragment(...)`
|
||||
- shared analytics detail now appends a `template-signal support-bias snapshot` section
|
||||
- the older live guard summary inside the support-kind split now reads `current support-bias ...` to distinguish it from the stored at-launch split
|
||||
|
||||
## Product effect
|
||||
|
||||
- operators can now inspect whether broader analytics picks launched with an already-earned support-bias lane are actually outperforming broader picks launched while that lane was still suppressed
|
||||
- older broader launches remain visible under `no-retained` instead of being silently folded into a fabricated earned or suppressed bucket
|
||||
- live `[Template Launch]` detail, template history, and recap all inherit the new repo-backed snapshot split automatically
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- repo-backed template analytics persist broader template-signal support-bias snapshot buckets for `earned`, `suppressed`, and `no-retained`
|
||||
- broader launches route by stored `TemplateSignalSupportBiasSnapshot.State`
|
||||
- shared analytics detail shows the new snapshot split
|
||||
- when both earned and suppressed lanes have outcome history, shared analytics detail reports `earned-minus-suppressed` success and mistake deltas
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. full `Development Editor|Win64` build succeeds
|
||||
2. confirm repository analytics route broader template-signal launches by stored support-bias snapshot state
|
||||
3. confirm older launches without a retained snapshot appear under `no-retained`
|
||||
4. confirm live detail, template history, and recap all show the new `template-signal support-bias snapshot` split through shared analytics detail
|
||||
Loading…
Add table
Reference in a new issue