Wire recognition runtime into replay and coaching flow

This commit is contained in:
axiomlogicnexus 2026-05-06 04:31:20 +02:00
parent 1860f1488e
commit 44ecb902a0
14 changed files with 1261 additions and 4 deletions

View file

@ -12,5 +12,6 @@ OpenSessionPath=/vision/session/open
SubmitFramePath=/vision/frame
CommitObservationPath=/vision/commit
FinalizeSessionPath=/vision/session/finalize
CloseSessionPath=/vision/session/close
HealthPath=/vision/health
RequestTimeoutSeconds=5.0

View file

@ -343,6 +343,27 @@ FHyperTwistVisionFinalizeResult UHyperTwistHttpVisionClient::FinalizeVisionSessi
return Result;
}
bool UHyperTwistHttpVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError)
{
OutError.Reset();
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath),
HyperTwistHttpVisionClientInternal::MakeSessionIdJson(SessionId),
Response))
{
OutError = Response.Error;
LastTransportError = OutError;
return false;
}
LastTransportError.Reset();
return true;
}
FHyperTwistVisionServiceHealth UHyperTwistHttpVisionClient::GetVisionServiceHealth() const
{
FHyperTwistVisionServiceHealth Health;

View file

@ -68,6 +68,21 @@ FHyperTwistVisionFinalizeResult UHyperTwistMockVisionClient::FinalizeVisionSessi
return Result;
}
bool UHyperTwistMockVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError)
{
OutError.Reset();
if (!SessionConfigs.Contains(SessionId))
{
OutError = TEXT("session-not-open");
return false;
}
SessionConfigs.Remove(SessionId);
SessionFrameCounts.Remove(SessionId);
return true;
}
FHyperTwistVisionServiceHealth UHyperTwistMockVisionClient::GetVisionServiceHealth() const
{
FHyperTwistVisionServiceHealth Health = UHyperTwistContractLibrary::MakeMockVisionServiceHealth();

View file

@ -27,6 +27,8 @@ void UHyperTwistTrainingPanelWidget::RefreshTrainingState()
{
CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this);
CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this);
CachedRecognitionSessionState =
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this);
UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState(
this,
CachedImportedRuntimeSelectionState
@ -129,6 +131,48 @@ void UHyperTwistTrainingPanelWidget::ClearActiveTrainingRun()
RefreshTrainingState();
}
bool UHyperTwistTrainingPanelWidget::OpenActiveRecognitionSession(FString& OutError)
{
const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError);
RefreshTrainingState();
return bOpened;
}
FHyperTwistVisionPreviewResult UHyperTwistTrainingPanelWidget::SubmitActiveRecognitionFrame(
const FHyperTwistVisionFrameEnvelope& Frame
)
{
const FHyperTwistVisionPreviewResult Result =
UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame);
RefreshTrainingState();
return Result;
}
FHyperTwistVisionCommitResult UHyperTwistTrainingPanelWidget::CommitActiveRecognitionObservation(
const FHyperTwistVisionCommitRequest& Request
)
{
const FHyperTwistVisionCommitResult Result =
UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request);
RefreshTrainingState();
return Result;
}
FHyperTwistVisionFinalizeResult UHyperTwistTrainingPanelWidget::FinalizeActiveRecognitionSession()
{
const FHyperTwistVisionFinalizeResult Result =
UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this);
RefreshTrainingState();
return Result;
}
bool UHyperTwistTrainingPanelWidget::CloseActiveRecognitionSession(FString& OutError)
{
const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError);
RefreshTrainingState();
return bClosed;
}
FHyperTwistTrainingRunState UHyperTwistTrainingPanelWidget::ApplyActiveImportedRuntimeSelectorOption(
const FString& SelectorId,
const FString& OptionId

View file

@ -231,6 +231,72 @@ FHyperTwistTrainingSessionSummary UHyperTwistTrainingRuntimeLibrary::GetActiveTr
return FHyperTwistTrainingSessionSummary();
}
bool UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(
UObject* WorldContextObject,
FString& OutError
)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->OpenActiveRecognitionSession(OutError);
}
OutError = TEXT("Training subsystem was unavailable.");
return false;
}
FHyperTwistVisionPreviewResult UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(
UObject* WorldContextObject,
const FHyperTwistVisionFrameEnvelope& Frame
)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->SubmitActiveRecognitionFrame(Frame);
}
return FHyperTwistVisionPreviewResult();
}
FHyperTwistVisionCommitResult UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(
UObject* WorldContextObject,
const FHyperTwistVisionCommitRequest& Request
)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->CommitActiveRecognitionObservation(Request);
}
return FHyperTwistVisionCommitResult();
}
FHyperTwistVisionFinalizeResult UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(
UObject* WorldContextObject
)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->FinalizeActiveRecognitionSession();
}
return FHyperTwistVisionFinalizeResult();
}
bool UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(
UObject* WorldContextObject,
FString& OutError
)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->CloseActiveRecognitionSession(OutError);
}
OutError = TEXT("Training subsystem was unavailable.");
return false;
}
FHyperTwistTrainingMethodDrillSummary UHyperTwistTrainingRuntimeLibrary::GetActiveMethodDrillSummary(
UObject* WorldContextObject
)
@ -450,6 +516,17 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingRuntimeLibrary::GetActive
return FHyperTwistRecognitionReplaySummary();
}
FHyperTwistTrainingRecognitionSessionState
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(UObject* WorldContextObject)
{
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
{
return TrainingSubsystem->GetActiveRecognitionSessionState();
}
return FHyperTwistTrainingRecognitionSessionState();
}
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingCoachSignals(
UObject* WorldContextObject
)

View file

@ -81,10 +81,54 @@ void AHyperTwistTrainingSessionActor::ClearRun()
RefreshCachedTrainingState();
}
bool AHyperTwistTrainingSessionActor::OpenActiveRecognitionSession(FString& OutError)
{
const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError);
RefreshCachedTrainingState();
return bOpened;
}
FHyperTwistVisionPreviewResult AHyperTwistTrainingSessionActor::SubmitActiveRecognitionFrame(
const FHyperTwistVisionFrameEnvelope& Frame
)
{
const FHyperTwistVisionPreviewResult Result =
UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame);
RefreshCachedTrainingState();
return Result;
}
FHyperTwistVisionCommitResult AHyperTwistTrainingSessionActor::CommitActiveRecognitionObservation(
const FHyperTwistVisionCommitRequest& Request
)
{
const FHyperTwistVisionCommitResult Result =
UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request);
RefreshCachedTrainingState();
return Result;
}
FHyperTwistVisionFinalizeResult AHyperTwistTrainingSessionActor::FinalizeActiveRecognitionSession()
{
const FHyperTwistVisionFinalizeResult Result =
UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this);
RefreshCachedTrainingState();
return Result;
}
bool AHyperTwistTrainingSessionActor::CloseActiveRecognitionSession(FString& OutError)
{
const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError);
RefreshCachedTrainingState();
return bClosed;
}
void AHyperTwistTrainingSessionActor::RefreshCachedTrainingState()
{
CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this);
CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this);
CachedRecognitionSessionState =
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this);
UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState(
this,
CachedImportedRuntimeSelectionState

View file

@ -1,12 +1,15 @@
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
#include "HyperTwistRecognition/HyperTwistVisionClient.h"
#include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h"
#include "HyperTwistReplay/HyperTwistReplayLibrary.h"
#include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingCatalogLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingCoachLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h"
#include "JsonObjectConverter.h"
namespace HyperTwistTrainingSubsystemInternal
{
@ -1144,6 +1147,119 @@ namespace HyperTwistTrainingSubsystemInternal
QueueEntry.SuggestedMode == EHyperTwistTrainingDeliveryMode::RecognitionAssisted;
return Brief;
}
template <typename TStruct>
bool SerializeStruct(const TStruct& Value, FString& OutJson)
{
return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0);
}
template <typename TStruct>
bool DeserializeStruct(const FString& Json, TStruct& OutValue)
{
return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0);
}
IHyperTwistVisionClient* ResolveVisionClient(UObject* Candidate)
{
return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistVisionClient::StaticClass())
? Cast<IHyperTwistVisionClient>(Candidate)
: nullptr;
}
FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
{
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
}
FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal)
{
return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal));
}
EHyperTwistPuzzleFamily ResolvePuzzleFamilyFromPuzzleId(const FString& PuzzleId)
{
if (PuzzleId.StartsWith(TEXT("hypercube/")))
{
return EHyperTwistPuzzleFamily::Hypercube;
}
if (PuzzleId.StartsWith(TEXT("cube/")))
{
return EHyperTwistPuzzleFamily::ClassicCube;
}
return EHyperTwistPuzzleFamily::Other;
}
FString ResolveNotationProfileFromPuzzleId(const FString& PuzzleId)
{
if (PuzzleId.StartsWith(TEXT("hypercube/")))
{
return TEXT("hyper-4d");
}
return TEXT("classic-wca");
}
bool IsRecognitionReplayEventType(const EHyperTwistReplayEventType EventType)
{
return EventType == EHyperTwistReplayEventType::RecognitionPreview
|| EventType == EHyperTwistReplayEventType::RecognitionScan
|| EventType == EHyperTwistReplayEventType::RecognitionCommit
|| EventType == EHyperTwistReplayEventType::RecognitionCorrection;
}
FHyperTwistReplayRecognitionPayload BuildFinalizeCorrectionPayload(
const FHyperTwistVisionFinalizeResult& FinalizeResult
)
{
FHyperTwistReplayRecognitionPayload Payload;
Payload.Snapshot = FinalizeResult.FinalSnapshot;
Payload.Confidence = FinalizeResult.ConfidenceRollup;
return Payload;
}
FHyperTwistReplayRecognitionPayload BuildFinalizeCommitPayload(
const FHyperTwistVisionFinalizeResult& FinalizeResult
)
{
FHyperTwistReplayRecognitionPayload Payload;
Payload.Snapshot = FinalizeResult.FinalSnapshot;
Payload.Confidence = FinalizeResult.ConfidenceRollup;
return Payload;
}
FHyperTwistStateSnapshot BuildSnapshotFromFinalState(
const FHyperTwistVisionFinalizeResult& FinalizeResult
)
{
FHyperTwistStateSnapshot Snapshot = FinalizeResult.FinalSnapshot;
if (Snapshot.SnapshotId.IsEmpty())
{
Snapshot.SnapshotId = FString::Printf(TEXT("recognition_final_%s"), *FinalizeResult.SessionId);
}
if (!Snapshot.State.IsStructurallyValid() && FinalizeResult.FinalState.IsStructurallyValid())
{
Snapshot.State = FinalizeResult.FinalState;
}
if (Snapshot.DerivedHash.IsEmpty())
{
Snapshot.DerivedHash = FString::Printf(TEXT("recognition-final-%s"), *FinalizeResult.SessionId);
}
return Snapshot;
}
int32 CountRecognitionCorrectionEvents(const TArray<FHyperTwistReplayEvent>& Events)
{
int32 Count = 0;
for (const FHyperTwistReplayEvent& Event : Events)
{
if (Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection)
{
++Count;
}
}
return Count;
}
}
bool UHyperTwistTrainingSubsystem::HasActiveRun() const
@ -2064,6 +2180,11 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingSubsystem::GetActiveRecog
return ActiveRecognitionReplaySummary;
}
FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActiveRecognitionSessionState() const
{
return ActiveRecognitionSessionState;
}
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const
{
return ActiveCoachSignals;
@ -3477,13 +3598,19 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe
EHyperTwistTrainingDeliveryMode Mode
)
{
if (ActiveRecognitionSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
ResetRecognitionSessionState();
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus();
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
ActiveCoachFollowUpBrief = FHyperTwistCoachBrief();
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
ActiveRunState = UHyperTwistTrainingLibrary::StartTrainingRun(Deck, UserId, SessionId, Mode);
ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest;
ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest;
bHasActiveRun = ActiveRunState.IsStructurallyValid();
RefreshSummary();
TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState(
@ -3736,6 +3863,12 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
return FHyperTwistTrainingSession();
}
if (ActiveRecognitionSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
ActiveRunState.Session = UHyperTwistTrainingLibrary::CompleteTrainingSession(ActiveRunState.Session, bAbortSession);
ActiveRunState.CurrentSelection = FHyperTwistTrainingCaseSelection();
ActiveRunState.RemainingCaseIds.Reset();
@ -3790,21 +3923,498 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
);
}
}
ResetRecognitionSessionState();
RefreshRepositoryViews();
return ActiveRunState.Session;
}
void UHyperTwistTrainingSubsystem::ClearActiveRun()
{
if (ActiveRecognitionSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
ActiveRunState = FHyperTwistTrainingRunState();
ActiveRunSummary = FHyperTwistTrainingSessionSummary();
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
bHasActiveRun = false;
ResetRecognitionSessionState();
RefreshRepositoryViews();
}
bool UHyperTwistTrainingSubsystem::OpenActiveRecognitionSession(FString& OutError)
{
OutError.Reset();
if (!HasActiveRun())
{
OutError = TEXT("no-active-training-run");
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
if (VisionClient == nullptr)
{
OutError = TEXT("recognition-client-unavailable");
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
const FHyperTwistVisionSessionConfig SessionConfig = BuildActiveRecognitionSessionConfig();
if (SessionConfig.SessionId.IsEmpty())
{
OutError = TEXT("recognition-session-id-unavailable");
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
if (ActiveRecognitionSessionState.bSessionOpen
&& ActiveRecognitionSessionState.ActiveSessionId == SessionConfig.SessionId)
{
ActiveRecognitionSessionState.SessionConfig = SessionConfig;
ActiveRecognitionSessionState.LastError.Reset();
ActiveRecognitionSessionState.ServiceHealth.LastError.Reset();
return true;
}
if (ActiveRecognitionSessionState.bSessionOpen
&& ActiveRecognitionSessionState.ActiveSessionId != SessionConfig.SessionId)
{
FString CloseError;
VisionClient->CloseVisionSession(ActiveRecognitionSessionState.ActiveSessionId, CloseError);
}
if (!VisionClient->OpenVisionSession(SessionConfig, OutError))
{
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveRecognitionSessionState.bSessionOpen = true;
ActiveRecognitionSessionState.bUsingProviderBackedClient =
ActiveRecognitionVisionClientObject != nullptr
&& ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>();
ActiveRecognitionSessionState.ActiveSessionId = SessionConfig.SessionId;
ActiveRecognitionSessionState.OpenedAtUtc = OpenedAtUtc;
ActiveRecognitionSessionState.LastUpdatedAtUtc = OpenedAtUtc;
ActiveRecognitionSessionState.LastError.Reset();
ActiveRecognitionSessionState.SessionConfig = SessionConfig;
ActiveRecognitionSessionState.bHasPreviewResult = false;
ActiveRecognitionSessionState.bHasCommitResult = false;
ActiveRecognitionSessionState.LastPreviewResult = FHyperTwistVisionPreviewResult();
ActiveRecognitionSessionState.LastCommitResult = FHyperTwistVisionCommitResult();
RefreshRecognitionServiceHealth();
return true;
}
FHyperTwistVisionPreviewResult UHyperTwistTrainingSubsystem::SubmitActiveRecognitionFrame(
const FHyperTwistVisionFrameEnvelope& Frame
)
{
FHyperTwistVisionPreviewResult Result;
if (!HasActiveRun())
{
Result.Guidance = TEXT("no-active-training-run");
return Result;
}
FString OpenError;
if (!OpenActiveRecognitionSession(OpenError))
{
Result.SessionId = !Frame.SessionId.IsEmpty() ? Frame.SessionId : BuildActiveRecognitionSessionConfig().SessionId;
Result.FrameId = Frame.FrameId;
Result.Guidance = OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError;
return Result;
}
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
if (VisionClient == nullptr)
{
Result.Guidance = TEXT("recognition-client-unavailable");
ActiveRecognitionSessionState.LastError = Result.Guidance;
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
return Result;
}
FHyperTwistVisionFrameEnvelope NormalizedFrame = Frame;
NormalizedFrame.SessionId = ActiveRecognitionSessionState.ActiveSessionId;
if (NormalizedFrame.FrameOrdinal <= 0)
{
NormalizedFrame.FrameOrdinal = ActiveRecognitionSessionState.SubmittedFrameCount + 1;
}
if (NormalizedFrame.FrameId.IsEmpty())
{
NormalizedFrame.FrameId = HyperTwistTrainingSubsystemInternal::BuildRecognitionFrameId(
NormalizedFrame.SessionId,
NormalizedFrame.FrameOrdinal
);
}
if (NormalizedFrame.CapturedAtUtc.IsEmpty())
{
NormalizedFrame.CapturedAtUtc = FDateTime::UtcNow().ToIso8601();
}
Result = VisionClient->SubmitVisionFrame(NormalizedFrame);
if (Result.SessionId.IsEmpty())
{
Result.SessionId = NormalizedFrame.SessionId;
}
if (Result.FrameId.IsEmpty())
{
Result.FrameId = NormalizedFrame.FrameId;
}
if (Result.PreviewState.State.SourceSessionId.IsEmpty())
{
Result.PreviewState.State.SourceSessionId = NormalizedFrame.SessionId;
}
ActiveRecognitionSessionState.bHasPreviewResult = true;
ActiveRecognitionSessionState.LastPreviewResult = Result;
ActiveRecognitionSessionState.SubmittedFrameCount = FMath::Max(
ActiveRecognitionSessionState.SubmittedFrameCount,
NormalizedFrame.FrameOrdinal
);
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveRecognitionSessionState.LastError.Reset();
ActiveRecognitionSessionState.ServiceHealth.LastError.Reset();
FHyperTwistReplayRecognitionPayload Payload;
Payload.Snapshot = Result.PreviewState;
Payload.Confidence = Result.PreviewConfidence;
Payload.ScanStage = NormalizedFrame.CameraPoseHint;
AppendRecognitionReplayEvent(
ActiveRecognitionSessionState.SubmittedFrameCount <= 1
? EHyperTwistReplayEventType::RecognitionPreview
: EHyperTwistReplayEventType::RecognitionScan,
Payload,
NormalizedFrame.SourceTimestampMs > 0 ? NormalizedFrame.SourceTimestampMs : INDEX_NONE
);
SynchronizeRecognitionReplayMutation();
return Result;
}
FHyperTwistVisionCommitResult UHyperTwistTrainingSubsystem::CommitActiveRecognitionObservation(
const FHyperTwistVisionCommitRequest& Request
)
{
FHyperTwistVisionCommitResult Result;
if (!HasActiveRun())
{
Result.Warnings.Add(TEXT("no-active-training-run"));
return Result;
}
FString OpenError;
if (!OpenActiveRecognitionSession(OpenError))
{
Result.SessionId = Request.SessionId;
Result.CommitKind = Request.CommitKind;
Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError);
return Result;
}
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
if (VisionClient == nullptr)
{
Result.Warnings.Add(TEXT("recognition-client-unavailable"));
ActiveRecognitionSessionState.LastError = Result.Warnings[0];
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
return Result;
}
FHyperTwistVisionCommitRequest NormalizedRequest = Request;
NormalizedRequest.SessionId = ActiveRecognitionSessionState.ActiveSessionId;
if (NormalizedRequest.CommitKind.IsEmpty())
{
NormalizedRequest.CommitKind = TEXT("face");
}
Result = VisionClient->CommitVisionObservation(NormalizedRequest);
if (Result.SessionId.IsEmpty())
{
Result.SessionId = NormalizedRequest.SessionId;
}
if (Result.CommitKind.IsEmpty())
{
Result.CommitKind = NormalizedRequest.CommitKind;
}
if (Result.Snapshot.State.SourceSessionId.IsEmpty())
{
Result.Snapshot.State.SourceSessionId = NormalizedRequest.SessionId;
}
const FString ResolvedStage = !Result.CommittedUnit.IsEmpty()
? Result.CommittedUnit
: (!NormalizedRequest.TargetStage.IsEmpty() ? NormalizedRequest.TargetStage : NormalizedRequest.TargetFace);
const bool bHasCorrectionSignals =
Result.Warnings.Num() > 0 || Result.MissingUnits.Num() > 0;
const bool bHasCommitSignal =
!Result.CommittedUnit.IsEmpty()
|| Result.Snapshot.IsStructurallyValid()
|| Result.Snapshot.State.IsStructurallyValid()
|| Result.Confidence > 0.0f;
if (bHasCorrectionSignals)
{
FHyperTwistReplayRecognitionPayload CorrectionPayload;
CorrectionPayload.Snapshot = Result.Snapshot;
CorrectionPayload.Confidence = Result.Confidence;
CorrectionPayload.CommittedFaceOrStage = ResolvedStage;
AppendRecognitionReplayEvent(
EHyperTwistReplayEventType::RecognitionCorrection,
CorrectionPayload,
INDEX_NONE
);
++ActiveRecognitionSessionState.CorrectionEventCount;
}
if (bHasCommitSignal)
{
FHyperTwistReplayRecognitionPayload CommitPayload;
CommitPayload.Snapshot = Result.Snapshot;
CommitPayload.Confidence = Result.Confidence;
CommitPayload.CommittedFaceOrStage = ResolvedStage;
AppendRecognitionReplayEvent(
EHyperTwistReplayEventType::RecognitionCommit,
CommitPayload,
INDEX_NONE
);
++ActiveRecognitionSessionState.CommittedObservationCount;
}
ActiveRecognitionSessionState.bHasCommitResult = true;
ActiveRecognitionSessionState.LastCommitResult = Result;
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveRecognitionSessionState.LastError =
Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString();
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
if (bHasCorrectionSignals || bHasCommitSignal)
{
SynchronizeRecognitionReplayMutation();
}
return Result;
}
FHyperTwistVisionFinalizeResult UHyperTwistTrainingSubsystem::FinalizeActiveRecognitionSession()
{
FHyperTwistVisionFinalizeResult Result;
if (!HasActiveRun())
{
Result.NormalizationWarnings.Add(TEXT("no-active-training-run"));
return Result;
}
if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty())
{
Result.NormalizationWarnings.Add(TEXT("recognition-session-not-open"));
return Result;
}
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
if (VisionClient == nullptr)
{
Result.NormalizationWarnings.Add(TEXT("recognition-client-unavailable"));
ActiveRecognitionSessionState.LastError = Result.NormalizationWarnings[0];
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
return Result;
}
const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId;
Result = VisionClient->FinalizeVisionSession(SessionId);
if (Result.SessionId.IsEmpty())
{
Result.SessionId = SessionId;
}
if (Result.FinalState.SourceSessionId.IsEmpty())
{
Result.FinalState.SourceSessionId = SessionId;
}
if (Result.FinalSnapshot.State.SourceSessionId.IsEmpty())
{
Result.FinalSnapshot.State.SourceSessionId = SessionId;
}
TArray<FHyperTwistReplayEvent> PendingRecognitionEvents;
for (const FString& SeedJson : Result.ReplaySeedEventsJson)
{
FHyperTwistReplayPacket SeedPacket;
if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedPacket))
{
for (const FHyperTwistReplayEvent& Event : SeedPacket.Events)
{
if (HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType))
{
PendingRecognitionEvents.Add(Event);
}
}
continue;
}
FHyperTwistReplayEvent SeedEvent;
if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedEvent)
&& HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(SeedEvent.EventType))
{
PendingRecognitionEvents.Add(SeedEvent);
}
}
const bool bHadProviderSeedEvents = PendingRecognitionEvents.Num() > 0;
const bool bHasProviderCorrectionEvent = PendingRecognitionEvents.ContainsByPredicate(
[](const FHyperTwistReplayEvent& Event)
{
return Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection;
}
);
if (Result.NormalizationWarnings.Num() > 0 && !bHasProviderCorrectionEvent)
{
FHyperTwistReplayRecognitionPayload CorrectionPayload =
HyperTwistTrainingSubsystemInternal::BuildFinalizeCorrectionPayload(Result);
CorrectionPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result);
FString CorrectionPayloadJson;
if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CorrectionPayload, CorrectionPayloadJson))
{
FHyperTwistReplayEvent CorrectionEvent;
CorrectionEvent.EventType = EHyperTwistReplayEventType::RecognitionCorrection;
CorrectionEvent.TimeMs = PendingRecognitionEvents.Num() > 0
? PendingRecognitionEvents.Last().TimeMs
: INDEX_NONE;
CorrectionEvent.PayloadJson = CorrectionPayloadJson;
const int32 InsertIndex = PendingRecognitionEvents.IndexOfByPredicate(
[](const FHyperTwistReplayEvent& Event)
{
return Event.EventType == EHyperTwistReplayEventType::RecognitionCommit;
}
);
if (InsertIndex == INDEX_NONE)
{
PendingRecognitionEvents.Add(CorrectionEvent);
}
else
{
PendingRecognitionEvents.Insert(CorrectionEvent, InsertIndex);
}
++ActiveRecognitionSessionState.CorrectionEventCount;
}
}
if (!bHadProviderSeedEvents)
{
FHyperTwistReplayRecognitionPayload CommitPayload =
HyperTwistTrainingSubsystemInternal::BuildFinalizeCommitPayload(Result);
CommitPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result);
FString CommitPayloadJson;
if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CommitPayload, CommitPayloadJson))
{
FHyperTwistReplayEvent CommitEvent;
CommitEvent.EventType = EHyperTwistReplayEventType::RecognitionCommit;
CommitEvent.PayloadJson = CommitPayloadJson;
PendingRecognitionEvents.Add(CommitEvent);
}
}
bool bMutatedReplay = false;
for (const FHyperTwistReplayEvent& Event : PendingRecognitionEvents)
{
if (!HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType))
{
continue;
}
ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent(
ActiveRunState.ReplayPacket,
Event
);
bMutatedReplay = true;
}
FString CloseError;
if (!VisionClient->CloseVisionSession(SessionId, CloseError) && CloseError != TEXT("session-not-open"))
{
Result.NormalizationWarnings.AddUnique(CloseError);
}
ActiveRecognitionSessionState.bSessionOpen = false;
ActiveRecognitionSessionState.bHasFinalizeResult = true;
ActiveRecognitionSessionState.LastFinalizeResult = Result;
ActiveRecognitionSessionState.ActiveSessionId.Reset();
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
++ActiveRecognitionSessionState.FinalizedSessionCount;
ActiveRecognitionSessionState.LastError =
Result.NormalizationWarnings.Num() > 0 ? FString::Join(Result.NormalizationWarnings, TEXT(" | ")) : FString();
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
if (bMutatedReplay)
{
SynchronizeRecognitionReplayMutation();
}
return Result;
}
bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutError)
{
OutError.Reset();
if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty())
{
RefreshRecognitionServiceHealth();
return true;
}
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
if (VisionClient == nullptr)
{
OutError = TEXT("recognition-client-unavailable");
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId;
const bool bClosed = VisionClient->CloseVisionSession(SessionId, OutError);
if (!bClosed && OutError != TEXT("session-not-open"))
{
ActiveRecognitionSessionState.LastError = OutError;
RefreshRecognitionServiceHealth();
return false;
}
ActiveRecognitionSessionState.bSessionOpen = false;
ActiveRecognitionSessionState.ActiveSessionId.Reset();
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveRecognitionSessionState.LastError.Reset();
RefreshRecognitionServiceHealth();
return true;
}
void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
{
ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState();
@ -3814,6 +4424,154 @@ void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
bHasActiveMethodDrillRun = false;
}
UObject* UHyperTwistTrainingSubsystem::ResolveRecognitionVisionClientObject()
{
LoadConfig();
const bool bUseMockClient = RecognitionClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase);
const bool bNeedsNewClient = ActiveRecognitionVisionClientObject == nullptr
|| (bUseMockClient && !ActiveRecognitionVisionClientObject->IsA<UHyperTwistMockVisionClient>())
|| (!bUseMockClient && !ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>());
if (bNeedsNewClient)
{
if (bUseMockClient)
{
ActiveRecognitionVisionClientObject = NewObject<UHyperTwistMockVisionClient>(this);
}
else
{
UHyperTwistHttpVisionClient* HttpVisionClient = NewObject<UHyperTwistHttpVisionClient>(this);
HttpVisionClient->LoadConfig();
ActiveRecognitionVisionClientObject = HttpVisionClient;
}
}
return ActiveRecognitionVisionClientObject;
}
FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognitionSessionConfig() const
{
FHyperTwistVisionSessionConfig SessionConfig = ActiveRecognitionSessionState.SessionConfig;
if (!HasActiveRun())
{
return SessionConfig;
}
const FHyperTwistPuzzleDefinitionRef& ReplayDefinition = ActiveRunState.ReplayPacket.PuzzleDefinition;
const FHyperTwistTrainingCase* ActiveCase = ActiveRunState.CurrentSelection.IsStructurallyValid()
? &ActiveRunState.CurrentSelection.TrainingCase
: HyperTwistTrainingSubsystemInternal::FindFirstStructurallyValidCase(ActiveRunState.ActiveDeck);
SessionConfig.SessionId = !ActiveRecognitionSessionState.ActiveSessionId.IsEmpty()
? ActiveRecognitionSessionState.ActiveSessionId
: HyperTwistTrainingSubsystemInternal::BuildRecognitionSessionId(
ActiveRunState.Session,
ActiveRecognitionSessionState.FinalizedSessionCount + 1
);
SessionConfig.PuzzleId = ReplayDefinition.IsStructurallyValid()
? ReplayDefinition.PuzzleId
: (ActiveCase != nullptr && !ActiveCase->PuzzleId.IsEmpty() ? ActiveCase->PuzzleId : TEXT("cube/3x3x3"));
SessionConfig.ExpectedPuzzleFamily = ReplayDefinition.IsStructurallyValid()
? ReplayDefinition.PuzzleFamily
: HyperTwistTrainingSubsystemInternal::ResolvePuzzleFamilyFromPuzzleId(SessionConfig.PuzzleId);
SessionConfig.NotationProfile = ReplayDefinition.IsStructurallyValid()
? ReplayDefinition.NotationProfile
: HyperTwistTrainingSubsystemInternal::ResolveNotationProfileFromPuzzleId(SessionConfig.PuzzleId);
SessionConfig.CaptureMode = EHyperTwistVisionCaptureMode::LiveCamera;
if (SessionConfig.FrameSize.X <= 0 || SessionConfig.FrameSize.Y <= 0)
{
SessionConfig.FrameSize = FIntPoint(1280, 720);
}
if (SessionConfig.MaxFrameRate <= 0)
{
SessionConfig.MaxFrameRate = 30;
}
return SessionConfig;
}
void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
{
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
ResolveRecognitionVisionClientObject()
);
ActiveRecognitionSessionState.bUsingProviderBackedClient =
ActiveRecognitionVisionClientObject != nullptr
&& ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>();
if (VisionClient == nullptr)
{
ActiveRecognitionSessionState.ServiceHealth = FHyperTwistVisionServiceHealth();
ActiveRecognitionSessionState.ServiceHealth.ProviderLabel = ActiveRecognitionSessionState.bUsingProviderBackedClient
? TEXT("local-http-sidecar")
: TEXT("mock");
ActiveRecognitionSessionState.ServiceHealth.ServiceEndpoint = ActiveRecognitionSessionState.bUsingProviderBackedClient
? TEXT("http://127.0.0.1:8766")
: TEXT("in-process://mock");
ActiveRecognitionSessionState.ServiceHealth.bReady = false;
ActiveRecognitionSessionState.ServiceHealth.LastError = TEXT("recognition-client-unavailable");
return;
}
ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth();
}
void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent(
const EHyperTwistReplayEventType EventType,
const FHyperTwistReplayRecognitionPayload& Payload,
const int32 PreferredTimeMs
)
{
if (!HasActiveRun())
{
return;
}
FString PayloadJson;
if (!HyperTwistTrainingSubsystemInternal::SerializeStruct(Payload, PayloadJson))
{
return;
}
FHyperTwistReplayEvent ReplayEvent;
ReplayEvent.EventType = EventType;
ReplayEvent.PayloadJson = PayloadJson;
const int32 LastTimeMs = ActiveRunState.ReplayPacket.Events.Num() > 0
? ActiveRunState.ReplayPacket.Events.Last().TimeMs
: 0;
const int32 BaseTimeMs = PreferredTimeMs >= 0 ? PreferredTimeMs : LastTimeMs + 1;
ReplayEvent.TimeMs = BaseTimeMs > LastTimeMs ? BaseTimeMs : LastTimeMs + 1;
ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent(
ActiveRunState.ReplayPacket,
ReplayEvent
);
}
void UHyperTwistTrainingSubsystem::SynchronizeRecognitionReplayMutation()
{
if (ActiveRunState.ReplayPacket.Events.Num() > 0)
{
ActiveRunState.LastReplayEvent = ActiveRunState.ReplayPacket.Events.Last();
}
else
{
ActiveRunState.LastReplayEvent = FHyperTwistReplayEvent();
}
RefreshSummary();
TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState(
TrainingRepositoryState,
ActiveRunState
);
RefreshRepositoryViews();
}
void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState()
{
ActiveRecognitionSessionState = FHyperTwistTrainingRecognitionSessionState();
RefreshRecognitionServiceHealth();
}
void UHyperTwistTrainingSubsystem::RefreshSummary()
{
if (!ActiveRunState.Session.IsStructurallyValid())

View file

@ -21,6 +21,7 @@ public:
virtual FHyperTwistVisionPreviewResult SubmitVisionFrame(const FHyperTwistVisionFrameEnvelope& Frame) = 0;
virtual FHyperTwistVisionCommitResult CommitVisionObservation(const FHyperTwistVisionCommitRequest& Request) = 0;
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) = 0;
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) = 0;
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const = 0;
};
@ -42,6 +43,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;
@ -77,6 +81,9 @@ public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString FinalizeSessionPath = TEXT("/vision/session/finalize");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString CloseSessionPath = TEXT("/vision/session/close");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString HealthPath = TEXT("/vision/health");
@ -98,6 +105,9 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;

View file

@ -53,6 +53,9 @@ public:
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training")
FHyperTwistTrainingRunStepResult LastStepResult;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition")
FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach")
TArray<FHyperTwistCoachSignal> CachedCoachSignals;
@ -136,6 +139,21 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
void ClearActiveTrainingRun();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool OpenActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool CloseActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Imported")
FHyperTwistTrainingRunState ApplyActiveImportedRuntimeSelectorOption(
const FString& SelectorId,

View file

@ -100,6 +100,27 @@ public:
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistTrainingSessionSummary GetActiveTrainingRunSummary(UObject* WorldContextObject);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static bool OpenActiveRecognitionSession(UObject* WorldContextObject, FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(
UObject* WorldContextObject,
const FHyperTwistVisionFrameEnvelope& Frame
);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(
UObject* WorldContextObject,
const FHyperTwistVisionCommitRequest& Request
);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(UObject* WorldContextObject);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static bool CloseActiveRecognitionSession(UObject* WorldContextObject, FString& OutError);
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Drill", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistTrainingMethodDrillSummary GetActiveMethodDrillSummary(UObject* WorldContextObject);
@ -173,6 +194,11 @@ public:
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistRecognitionReplaySummary GetActiveTrainingRecognitionReplaySummary(UObject* WorldContextObject);
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
static FHyperTwistTrainingRecognitionSessionState GetActiveTrainingRecognitionSessionState(
UObject* WorldContextObject
);
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach", meta = (WorldContext = "WorldContextObject"))
static TArray<FHyperTwistCoachSignal> GetActiveTrainingCoachSignals(UObject* WorldContextObject);

View file

@ -40,6 +40,9 @@ public:
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training")
FHyperTwistTrainingRunStepResult LastStepResult;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition")
FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach")
TArray<FHyperTwistCoachSignal> CachedCoachSignals;
@ -117,6 +120,21 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
void ClearRun();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool OpenActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool CloseActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
void RefreshCachedTrainingState();

View file

@ -8,7 +8,7 @@
#include "HyperTwistTraining/HyperTwistTrainingTypes.h"
#include "HyperTwistTrainingSubsystem.generated.h"
UCLASS()
UCLASS(Config = Game, DefaultConfig)
class UNREALHYPERTWIST_API UHyperTwistTrainingSubsystem : public UGameInstanceSubsystem
{
GENERATED_BODY()
@ -146,6 +146,9 @@ public:
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training")
FHyperTwistRecognitionReplaySummary GetActiveRecognitionReplaySummary() const;
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition")
FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const;
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach")
TArray<FHyperTwistCoachSignal> GetActiveCoachSignals() const;
@ -542,10 +545,35 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
void ClearActiveRun();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool OpenActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool CloseActiveRecognitionSession(FString& OutError);
private:
void RefreshSummary();
void RefreshMethodDrillSummary();
void RefreshRepositoryViews();
FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const;
UObject* ResolveRecognitionVisionClientObject();
void RefreshRecognitionServiceHealth();
void AppendRecognitionReplayEvent(
EHyperTwistReplayEventType EventType,
const FHyperTwistReplayRecognitionPayload& Payload,
int32 PreferredTimeMs
);
void SynchronizeRecognitionReplayMutation();
void ResetRecognitionSessionState();
TArray<FHyperTwistTrainingSessionTemplate> BuildTrainingSessionTemplatesForUser(
const FString& UserId,
const FString& ReferenceUtc
@ -611,6 +639,9 @@ private:
UPROPERTY()
FHyperTwistRecognitionReplaySummary ActiveRecognitionReplaySummary;
UPROPERTY()
FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState;
UPROPERTY()
TArray<FHyperTwistCoachSignal> ActiveCoachSignals;
@ -683,6 +714,12 @@ private:
UPROPERTY()
FHyperTwistTrainingReviewProgramSummary ActiveReviewProgramSummary;
UPROPERTY(Config)
FString RecognitionClientKind = TEXT("http-sidecar");
UPROPERTY()
TObjectPtr<UObject> ActiveRecognitionVisionClientObject;
UPROPERTY()
bool bHasActiveRun = false;

View file

@ -2,6 +2,7 @@
#include "CoreMinimal.h"
#include "HyperTwistCore/HyperTwistCoreTypes.h"
#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h"
#include "HyperTwistReplay/HyperTwistReplayTypes.h"
#include "HyperTwistTrainingTypes.generated.h"
@ -1077,6 +1078,66 @@ struct FHyperTwistTrainingRunState
}
};
USTRUCT(BlueprintType)
struct FHyperTwistTrainingRecognitionSessionState
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bSessionOpen = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bUsingProviderBackedClient = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bHasPreviewResult = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bHasCommitResult = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bHasFinalizeResult = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ActiveSessionId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString OpenedAtUtc;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastUpdatedAtUtc;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastError;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SubmittedFrameCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 CommittedObservationCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 CorrectionEventCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 FinalizedSessionCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistVisionSessionConfig SessionConfig;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistVisionServiceHealth ServiceHealth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistVisionPreviewResult LastPreviewResult;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistVisionCommitResult LastCommitResult;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistVisionFinalizeResult LastFinalizeResult;
};
USTRUCT(BlueprintType)
struct FHyperTwistTrainingRunStepResult
{

View file

@ -0,0 +1,127 @@
# HyperTwist Phase 4 recognition runtime handoff packet
Created on `2026-05-06`
Status:
- first-party HyperTwist packet
- bounded Phase `4` recognition runtime-handoff slice
## Purpose
This packet closes the first honest runtime handoff after the provider-backed recognition client landed.
The open task is:
- move recognition from a sidecar contract plus standalone client into owned training runtime state, replay state, and coaching refresh flow
It is not:
- a large recognition UI pass
- a websocket/live-stream rewrite
- a speech or simulation packet
- a broad refactor of the training subsystem
## Scope
Bounded lane:
- complete recognition session lifecycle ownership in `HyperTwistTraining`
- expose runtime open / preview / commit / finalize / close surfaces through the subsystem, runtime library, panel widget, and session actor
- append normalized recognition replay events into the active run packet
- refresh repository/coaching derived state after replay mutation so existing replay-to-coach logic can wake up on live recognition data
Out of scope:
- broad dashboard redesign
- provider-specific protocol expansion beyond the current HTTP sidecar contract
- new gameplay content or speech flows
- exact puzzle-state engine deepening
## Why this was the right next packet
Before this slice:
- the repo had a provider-backed recognition client
- replay review analytics and recognition replay summaries already existed
- coach-signal derivation already reacted to recognition replay ambiguity and correction pressure
But recognition still did not flow through the owned training runtime.
So the missing bridge was:
- own the recognition session in `HyperTwistTraining`
- append replay events from recognition runtime activity
- let the existing replay/training/coaching stack consume those events without inventing a second parallel lane
## What landed
Primary code changes:
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h`
- added explicit `CloseVisionSession(...)` to the owned vision-client contract
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp`
- `UnrealHyperTwist/Config/DefaultGame.ini`
- completed close-session support for both mock and provider-backed client implementations
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h`
- added `FHyperTwistTrainingRecognitionSessionState`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp`
- added owned runtime methods for:
- open active recognition session
- submit active recognition frame
- commit active recognition observation
- finalize active recognition session
- close active recognition session
- added owned recognition session state, provider-health refresh, replay-event append, and replay-mutation synchronization helpers
- recognition replay mutation now records run state and refreshes repository/coaching views in the same subsystem pass
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp`
- exposed the new recognition runtime lane through the existing world-context runtime surface
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp`
- added thin runtime-consumer wrappers and cached recognition session state
## Runtime behavior now
The active training runtime now owns:
- recognition session open/closed state
- provider-health inspection state
- last preview / commit / finalize results
- cumulative frame / commit / correction / finalize counters
Recognition runtime activity now mutates the active replay packet through owned event types:
- first frame -> `RecognitionPreview`
- later frames -> `RecognitionScan`
- commit warnings / missing units -> `RecognitionCorrection`
- commit results -> `RecognitionCommit`
- finalize outputs -> replay-seed recognition events when provided, otherwise a synthesized final commit
After replay mutation:
- the active run summary is refreshed
- run state is persisted back into the training repository
- repository-derived replay review analytics, recognition replay summaries, and coach signals are rebuilt in the existing same-pass flow
## Acceptance criteria
- recognition session lifecycle is now owned by first-party training runtime code
- provider-backed and mock clients both satisfy the full open/submit/commit/finalize/close contract
- recognition runtime events reach the active replay packet
- replay mutation wakes up the existing repository/coaching refresh path
- the panel widget and session actor can consume the runtime lane without bypassing the subsystem
- full product build succeeds
## Validation checklist
1. build `UnrealHyperTwist.sln` / `UnrealHyperTwistEditor`
2. confirm the subsystem, runtime library, panel widget, and session actor all compile together
3. confirm no Phase `3` generated-mode packet was reopened
4. confirm the provider-backed client lane remains bounded to the owned vision contract
That is the packet.