diff --git a/UnrealHyperTwist/Config/DefaultGame.ini b/UnrealHyperTwist/Config/DefaultGame.ini index b186534..da77545 100644 --- a/UnrealHyperTwist/Config/DefaultGame.ini +++ b/UnrealHyperTwist/Config/DefaultGame.ini @@ -12,5 +12,6 @@ OpenSessionPath=/vision/session/open SubmitFramePath=/vision/frame CommitObservationPath=/vision/commit FinalizeSessionPath=/vision/session/finalize +CloseSessionPath=/vision/session/close HealthPath=/vision/health RequestTimeoutSeconds=5.0 diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp index 1a718eb..d3cae55 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp @@ -343,6 +343,27 @@ FHyperTwistVisionFinalizeResult UHyperTwistHttpVisionClient::FinalizeVisionSessi return Result; } +bool UHyperTwistHttpVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError) +{ + OutError.Reset(); + + HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response; + if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest( + *this, + TEXT("POST"), + HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath), + HyperTwistHttpVisionClientInternal::MakeSessionIdJson(SessionId), + Response)) + { + OutError = Response.Error; + LastTransportError = OutError; + return false; + } + + LastTransportError.Reset(); + return true; +} + FHyperTwistVisionServiceHealth UHyperTwistHttpVisionClient::GetVisionServiceHealth() const { FHyperTwistVisionServiceHealth Health; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp index ea7c648..b3b2a66 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp @@ -68,6 +68,21 @@ FHyperTwistVisionFinalizeResult UHyperTwistMockVisionClient::FinalizeVisionSessi return Result; } +bool UHyperTwistMockVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError) +{ + OutError.Reset(); + + if (!SessionConfigs.Contains(SessionId)) + { + OutError = TEXT("session-not-open"); + return false; + } + + SessionConfigs.Remove(SessionId); + SessionFrameCounts.Remove(SessionId); + return true; +} + FHyperTwistVisionServiceHealth UHyperTwistMockVisionClient::GetVisionServiceHealth() const { FHyperTwistVisionServiceHealth Health = UHyperTwistContractLibrary::MakeMockVisionServiceHealth(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp index e3e3ab7..5216932 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp @@ -27,6 +27,8 @@ void UHyperTwistTrainingPanelWidget::RefreshTrainingState() { CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this); CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this); + CachedRecognitionSessionState = + UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this); UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState( this, CachedImportedRuntimeSelectionState @@ -129,6 +131,48 @@ void UHyperTwistTrainingPanelWidget::ClearActiveTrainingRun() RefreshTrainingState(); } +bool UHyperTwistTrainingPanelWidget::OpenActiveRecognitionSession(FString& OutError) +{ + const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError); + RefreshTrainingState(); + return bOpened; +} + +FHyperTwistVisionPreviewResult UHyperTwistTrainingPanelWidget::SubmitActiveRecognitionFrame( + const FHyperTwistVisionFrameEnvelope& Frame +) +{ + const FHyperTwistVisionPreviewResult Result = + UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame); + RefreshTrainingState(); + return Result; +} + +FHyperTwistVisionCommitResult UHyperTwistTrainingPanelWidget::CommitActiveRecognitionObservation( + const FHyperTwistVisionCommitRequest& Request +) +{ + const FHyperTwistVisionCommitResult Result = + UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request); + RefreshTrainingState(); + return Result; +} + +FHyperTwistVisionFinalizeResult UHyperTwistTrainingPanelWidget::FinalizeActiveRecognitionSession() +{ + const FHyperTwistVisionFinalizeResult Result = + UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this); + RefreshTrainingState(); + return Result; +} + +bool UHyperTwistTrainingPanelWidget::CloseActiveRecognitionSession(FString& OutError) +{ + const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError); + RefreshTrainingState(); + return bClosed; +} + FHyperTwistTrainingRunState UHyperTwistTrainingPanelWidget::ApplyActiveImportedRuntimeSelectorOption( const FString& SelectorId, const FString& OptionId diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp index c88d8cb..9cdf47b 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp @@ -231,6 +231,72 @@ FHyperTwistTrainingSessionSummary UHyperTwistTrainingRuntimeLibrary::GetActiveTr return FHyperTwistTrainingSessionSummary(); } +bool UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession( + UObject* WorldContextObject, + FString& OutError +) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->OpenActiveRecognitionSession(OutError); + } + + OutError = TEXT("Training subsystem was unavailable."); + return false; +} + +FHyperTwistVisionPreviewResult UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame( + UObject* WorldContextObject, + const FHyperTwistVisionFrameEnvelope& Frame +) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->SubmitActiveRecognitionFrame(Frame); + } + + return FHyperTwistVisionPreviewResult(); +} + +FHyperTwistVisionCommitResult UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation( + UObject* WorldContextObject, + const FHyperTwistVisionCommitRequest& Request +) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->CommitActiveRecognitionObservation(Request); + } + + return FHyperTwistVisionCommitResult(); +} + +FHyperTwistVisionFinalizeResult UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession( + UObject* WorldContextObject +) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->FinalizeActiveRecognitionSession(); + } + + return FHyperTwistVisionFinalizeResult(); +} + +bool UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession( + UObject* WorldContextObject, + FString& OutError +) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->CloseActiveRecognitionSession(OutError); + } + + OutError = TEXT("Training subsystem was unavailable."); + return false; +} + FHyperTwistTrainingMethodDrillSummary UHyperTwistTrainingRuntimeLibrary::GetActiveMethodDrillSummary( UObject* WorldContextObject ) @@ -450,6 +516,17 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingRuntimeLibrary::GetActive return FHyperTwistRecognitionReplaySummary(); } +FHyperTwistTrainingRecognitionSessionState +UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(UObject* WorldContextObject) +{ + if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject)) + { + return TrainingSubsystem->GetActiveRecognitionSessionState(); + } + + return FHyperTwistTrainingRecognitionSessionState(); +} + TArray UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingCoachSignals( UObject* WorldContextObject ) diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp index 62b553b..b0c3d10 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp @@ -81,10 +81,54 @@ void AHyperTwistTrainingSessionActor::ClearRun() RefreshCachedTrainingState(); } +bool AHyperTwistTrainingSessionActor::OpenActiveRecognitionSession(FString& OutError) +{ + const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError); + RefreshCachedTrainingState(); + return bOpened; +} + +FHyperTwistVisionPreviewResult AHyperTwistTrainingSessionActor::SubmitActiveRecognitionFrame( + const FHyperTwistVisionFrameEnvelope& Frame +) +{ + const FHyperTwistVisionPreviewResult Result = + UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame); + RefreshCachedTrainingState(); + return Result; +} + +FHyperTwistVisionCommitResult AHyperTwistTrainingSessionActor::CommitActiveRecognitionObservation( + const FHyperTwistVisionCommitRequest& Request +) +{ + const FHyperTwistVisionCommitResult Result = + UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request); + RefreshCachedTrainingState(); + return Result; +} + +FHyperTwistVisionFinalizeResult AHyperTwistTrainingSessionActor::FinalizeActiveRecognitionSession() +{ + const FHyperTwistVisionFinalizeResult Result = + UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this); + RefreshCachedTrainingState(); + return Result; +} + +bool AHyperTwistTrainingSessionActor::CloseActiveRecognitionSession(FString& OutError) +{ + const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError); + RefreshCachedTrainingState(); + return bClosed; +} + void AHyperTwistTrainingSessionActor::RefreshCachedTrainingState() { CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this); CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this); + CachedRecognitionSessionState = + UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this); UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState( this, CachedImportedRuntimeSelectionState diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp index c1fe54a..005d1c7 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp @@ -1,12 +1,15 @@ #include "HyperTwistTraining/HyperTwistTrainingSubsystem.h" +#include "HyperTwistRecognition/HyperTwistVisionClient.h" #include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h" +#include "HyperTwistReplay/HyperTwistReplayLibrary.h" #include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingCatalogLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingCoachLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h" +#include "JsonObjectConverter.h" namespace HyperTwistTrainingSubsystemInternal { @@ -1144,6 +1147,119 @@ namespace HyperTwistTrainingSubsystemInternal QueueEntry.SuggestedMode == EHyperTwistTrainingDeliveryMode::RecognitionAssisted; return Brief; } + + template + bool SerializeStruct(const TStruct& Value, FString& OutJson) + { + return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0); + } + + template + bool DeserializeStruct(const FString& Json, TStruct& OutValue) + { + return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0); + } + + IHyperTwistVisionClient* ResolveVisionClient(UObject* Candidate) + { + return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistVisionClient::StaticClass()) + ? Cast(Candidate) + : nullptr; + } + + FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal) + { + const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal); + return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal); + } + + FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal) + { + return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal)); + } + + EHyperTwistPuzzleFamily ResolvePuzzleFamilyFromPuzzleId(const FString& PuzzleId) + { + if (PuzzleId.StartsWith(TEXT("hypercube/"))) + { + return EHyperTwistPuzzleFamily::Hypercube; + } + if (PuzzleId.StartsWith(TEXT("cube/"))) + { + return EHyperTwistPuzzleFamily::ClassicCube; + } + return EHyperTwistPuzzleFamily::Other; + } + + FString ResolveNotationProfileFromPuzzleId(const FString& PuzzleId) + { + if (PuzzleId.StartsWith(TEXT("hypercube/"))) + { + return TEXT("hyper-4d"); + } + return TEXT("classic-wca"); + } + + bool IsRecognitionReplayEventType(const EHyperTwistReplayEventType EventType) + { + return EventType == EHyperTwistReplayEventType::RecognitionPreview + || EventType == EHyperTwistReplayEventType::RecognitionScan + || EventType == EHyperTwistReplayEventType::RecognitionCommit + || EventType == EHyperTwistReplayEventType::RecognitionCorrection; + } + + FHyperTwistReplayRecognitionPayload BuildFinalizeCorrectionPayload( + const FHyperTwistVisionFinalizeResult& FinalizeResult + ) + { + FHyperTwistReplayRecognitionPayload Payload; + Payload.Snapshot = FinalizeResult.FinalSnapshot; + Payload.Confidence = FinalizeResult.ConfidenceRollup; + return Payload; + } + + FHyperTwistReplayRecognitionPayload BuildFinalizeCommitPayload( + const FHyperTwistVisionFinalizeResult& FinalizeResult + ) + { + FHyperTwistReplayRecognitionPayload Payload; + Payload.Snapshot = FinalizeResult.FinalSnapshot; + Payload.Confidence = FinalizeResult.ConfidenceRollup; + return Payload; + } + + FHyperTwistStateSnapshot BuildSnapshotFromFinalState( + const FHyperTwistVisionFinalizeResult& FinalizeResult + ) + { + FHyperTwistStateSnapshot Snapshot = FinalizeResult.FinalSnapshot; + if (Snapshot.SnapshotId.IsEmpty()) + { + Snapshot.SnapshotId = FString::Printf(TEXT("recognition_final_%s"), *FinalizeResult.SessionId); + } + if (!Snapshot.State.IsStructurallyValid() && FinalizeResult.FinalState.IsStructurallyValid()) + { + Snapshot.State = FinalizeResult.FinalState; + } + if (Snapshot.DerivedHash.IsEmpty()) + { + Snapshot.DerivedHash = FString::Printf(TEXT("recognition-final-%s"), *FinalizeResult.SessionId); + } + return Snapshot; + } + + int32 CountRecognitionCorrectionEvents(const TArray& Events) + { + int32 Count = 0; + for (const FHyperTwistReplayEvent& Event : Events) + { + if (Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection) + { + ++Count; + } + } + return Count; + } } bool UHyperTwistTrainingSubsystem::HasActiveRun() const @@ -2064,6 +2180,11 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingSubsystem::GetActiveRecog return ActiveRecognitionReplaySummary; } +FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActiveRecognitionSessionState() const +{ + return ActiveRecognitionSessionState; +} + TArray UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const { return ActiveCoachSignals; @@ -3477,13 +3598,19 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe EHyperTwistTrainingDeliveryMode Mode ) { + if (ActiveRecognitionSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveRecognitionSession(CloseError); + } + ResetRecognitionSessionState(); ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState(); ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus(); ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan(); ActiveCoachFollowUpBrief = FHyperTwistCoachBrief(); - ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest(); + ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest(); ActiveRunState = UHyperTwistTrainingLibrary::StartTrainingRun(Deck, UserId, SessionId, Mode); - ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest; + ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest; bHasActiveRun = ActiveRunState.IsStructurallyValid(); RefreshSummary(); TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState( @@ -3736,6 +3863,12 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool return FHyperTwistTrainingSession(); } + if (ActiveRecognitionSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveRecognitionSession(CloseError); + } + ActiveRunState.Session = UHyperTwistTrainingLibrary::CompleteTrainingSession(ActiveRunState.Session, bAbortSession); ActiveRunState.CurrentSelection = FHyperTwistTrainingCaseSelection(); ActiveRunState.RemainingCaseIds.Reset(); @@ -3790,21 +3923,498 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool ); } } + ResetRecognitionSessionState(); RefreshRepositoryViews(); return ActiveRunState.Session; } void UHyperTwistTrainingSubsystem::ClearActiveRun() { + if (ActiveRecognitionSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveRecognitionSession(CloseError); + } ActiveRunState = FHyperTwistTrainingRunState(); ActiveRunSummary = FHyperTwistTrainingSessionSummary(); - ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest(); + ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest(); ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState(); ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan(); bHasActiveRun = false; + ResetRecognitionSessionState(); RefreshRepositoryViews(); } +bool UHyperTwistTrainingSubsystem::OpenActiveRecognitionSession(FString& OutError) +{ + OutError.Reset(); + + if (!HasActiveRun()) + { + OutError = TEXT("no-active-training-run"); + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + if (VisionClient == nullptr) + { + OutError = TEXT("recognition-client-unavailable"); + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + const FHyperTwistVisionSessionConfig SessionConfig = BuildActiveRecognitionSessionConfig(); + if (SessionConfig.SessionId.IsEmpty()) + { + OutError = TEXT("recognition-session-id-unavailable"); + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + if (ActiveRecognitionSessionState.bSessionOpen + && ActiveRecognitionSessionState.ActiveSessionId == SessionConfig.SessionId) + { + ActiveRecognitionSessionState.SessionConfig = SessionConfig; + ActiveRecognitionSessionState.LastError.Reset(); + ActiveRecognitionSessionState.ServiceHealth.LastError.Reset(); + return true; + } + + if (ActiveRecognitionSessionState.bSessionOpen + && ActiveRecognitionSessionState.ActiveSessionId != SessionConfig.SessionId) + { + FString CloseError; + VisionClient->CloseVisionSession(ActiveRecognitionSessionState.ActiveSessionId, CloseError); + } + + if (!VisionClient->OpenVisionSession(SessionConfig, OutError)) + { + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveRecognitionSessionState.bSessionOpen = true; + ActiveRecognitionSessionState.bUsingProviderBackedClient = + ActiveRecognitionVisionClientObject != nullptr + && ActiveRecognitionVisionClientObject->IsA(); + ActiveRecognitionSessionState.ActiveSessionId = SessionConfig.SessionId; + ActiveRecognitionSessionState.OpenedAtUtc = OpenedAtUtc; + ActiveRecognitionSessionState.LastUpdatedAtUtc = OpenedAtUtc; + ActiveRecognitionSessionState.LastError.Reset(); + ActiveRecognitionSessionState.SessionConfig = SessionConfig; + ActiveRecognitionSessionState.bHasPreviewResult = false; + ActiveRecognitionSessionState.bHasCommitResult = false; + ActiveRecognitionSessionState.LastPreviewResult = FHyperTwistVisionPreviewResult(); + ActiveRecognitionSessionState.LastCommitResult = FHyperTwistVisionCommitResult(); + RefreshRecognitionServiceHealth(); + return true; +} + +FHyperTwistVisionPreviewResult UHyperTwistTrainingSubsystem::SubmitActiveRecognitionFrame( + const FHyperTwistVisionFrameEnvelope& Frame +) +{ + FHyperTwistVisionPreviewResult Result; + + if (!HasActiveRun()) + { + Result.Guidance = TEXT("no-active-training-run"); + return Result; + } + + FString OpenError; + if (!OpenActiveRecognitionSession(OpenError)) + { + Result.SessionId = !Frame.SessionId.IsEmpty() ? Frame.SessionId : BuildActiveRecognitionSessionConfig().SessionId; + Result.FrameId = Frame.FrameId; + Result.Guidance = OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError; + return Result; + } + + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + if (VisionClient == nullptr) + { + Result.Guidance = TEXT("recognition-client-unavailable"); + ActiveRecognitionSessionState.LastError = Result.Guidance; + ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError; + return Result; + } + + FHyperTwistVisionFrameEnvelope NormalizedFrame = Frame; + NormalizedFrame.SessionId = ActiveRecognitionSessionState.ActiveSessionId; + if (NormalizedFrame.FrameOrdinal <= 0) + { + NormalizedFrame.FrameOrdinal = ActiveRecognitionSessionState.SubmittedFrameCount + 1; + } + if (NormalizedFrame.FrameId.IsEmpty()) + { + NormalizedFrame.FrameId = HyperTwistTrainingSubsystemInternal::BuildRecognitionFrameId( + NormalizedFrame.SessionId, + NormalizedFrame.FrameOrdinal + ); + } + if (NormalizedFrame.CapturedAtUtc.IsEmpty()) + { + NormalizedFrame.CapturedAtUtc = FDateTime::UtcNow().ToIso8601(); + } + + Result = VisionClient->SubmitVisionFrame(NormalizedFrame); + if (Result.SessionId.IsEmpty()) + { + Result.SessionId = NormalizedFrame.SessionId; + } + if (Result.FrameId.IsEmpty()) + { + Result.FrameId = NormalizedFrame.FrameId; + } + if (Result.PreviewState.State.SourceSessionId.IsEmpty()) + { + Result.PreviewState.State.SourceSessionId = NormalizedFrame.SessionId; + } + + ActiveRecognitionSessionState.bHasPreviewResult = true; + ActiveRecognitionSessionState.LastPreviewResult = Result; + ActiveRecognitionSessionState.SubmittedFrameCount = FMath::Max( + ActiveRecognitionSessionState.SubmittedFrameCount, + NormalizedFrame.FrameOrdinal + ); + ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveRecognitionSessionState.LastError.Reset(); + ActiveRecognitionSessionState.ServiceHealth.LastError.Reset(); + + FHyperTwistReplayRecognitionPayload Payload; + Payload.Snapshot = Result.PreviewState; + Payload.Confidence = Result.PreviewConfidence; + Payload.ScanStage = NormalizedFrame.CameraPoseHint; + AppendRecognitionReplayEvent( + ActiveRecognitionSessionState.SubmittedFrameCount <= 1 + ? EHyperTwistReplayEventType::RecognitionPreview + : EHyperTwistReplayEventType::RecognitionScan, + Payload, + NormalizedFrame.SourceTimestampMs > 0 ? NormalizedFrame.SourceTimestampMs : INDEX_NONE + ); + SynchronizeRecognitionReplayMutation(); + return Result; +} + +FHyperTwistVisionCommitResult UHyperTwistTrainingSubsystem::CommitActiveRecognitionObservation( + const FHyperTwistVisionCommitRequest& Request +) +{ + FHyperTwistVisionCommitResult Result; + + if (!HasActiveRun()) + { + Result.Warnings.Add(TEXT("no-active-training-run")); + return Result; + } + + FString OpenError; + if (!OpenActiveRecognitionSession(OpenError)) + { + Result.SessionId = Request.SessionId; + Result.CommitKind = Request.CommitKind; + Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError); + return Result; + } + + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + if (VisionClient == nullptr) + { + Result.Warnings.Add(TEXT("recognition-client-unavailable")); + ActiveRecognitionSessionState.LastError = Result.Warnings[0]; + ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError; + return Result; + } + + FHyperTwistVisionCommitRequest NormalizedRequest = Request; + NormalizedRequest.SessionId = ActiveRecognitionSessionState.ActiveSessionId; + if (NormalizedRequest.CommitKind.IsEmpty()) + { + NormalizedRequest.CommitKind = TEXT("face"); + } + + Result = VisionClient->CommitVisionObservation(NormalizedRequest); + if (Result.SessionId.IsEmpty()) + { + Result.SessionId = NormalizedRequest.SessionId; + } + if (Result.CommitKind.IsEmpty()) + { + Result.CommitKind = NormalizedRequest.CommitKind; + } + if (Result.Snapshot.State.SourceSessionId.IsEmpty()) + { + Result.Snapshot.State.SourceSessionId = NormalizedRequest.SessionId; + } + + const FString ResolvedStage = !Result.CommittedUnit.IsEmpty() + ? Result.CommittedUnit + : (!NormalizedRequest.TargetStage.IsEmpty() ? NormalizedRequest.TargetStage : NormalizedRequest.TargetFace); + const bool bHasCorrectionSignals = + Result.Warnings.Num() > 0 || Result.MissingUnits.Num() > 0; + const bool bHasCommitSignal = + !Result.CommittedUnit.IsEmpty() + || Result.Snapshot.IsStructurallyValid() + || Result.Snapshot.State.IsStructurallyValid() + || Result.Confidence > 0.0f; + + if (bHasCorrectionSignals) + { + FHyperTwistReplayRecognitionPayload CorrectionPayload; + CorrectionPayload.Snapshot = Result.Snapshot; + CorrectionPayload.Confidence = Result.Confidence; + CorrectionPayload.CommittedFaceOrStage = ResolvedStage; + AppendRecognitionReplayEvent( + EHyperTwistReplayEventType::RecognitionCorrection, + CorrectionPayload, + INDEX_NONE + ); + ++ActiveRecognitionSessionState.CorrectionEventCount; + } + + if (bHasCommitSignal) + { + FHyperTwistReplayRecognitionPayload CommitPayload; + CommitPayload.Snapshot = Result.Snapshot; + CommitPayload.Confidence = Result.Confidence; + CommitPayload.CommittedFaceOrStage = ResolvedStage; + AppendRecognitionReplayEvent( + EHyperTwistReplayEventType::RecognitionCommit, + CommitPayload, + INDEX_NONE + ); + ++ActiveRecognitionSessionState.CommittedObservationCount; + } + + ActiveRecognitionSessionState.bHasCommitResult = true; + ActiveRecognitionSessionState.LastCommitResult = Result; + ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveRecognitionSessionState.LastError = + Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString(); + ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError; + + if (bHasCorrectionSignals || bHasCommitSignal) + { + SynchronizeRecognitionReplayMutation(); + } + + return Result; +} + +FHyperTwistVisionFinalizeResult UHyperTwistTrainingSubsystem::FinalizeActiveRecognitionSession() +{ + FHyperTwistVisionFinalizeResult Result; + + if (!HasActiveRun()) + { + Result.NormalizationWarnings.Add(TEXT("no-active-training-run")); + return Result; + } + + if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty()) + { + Result.NormalizationWarnings.Add(TEXT("recognition-session-not-open")); + return Result; + } + + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + if (VisionClient == nullptr) + { + Result.NormalizationWarnings.Add(TEXT("recognition-client-unavailable")); + ActiveRecognitionSessionState.LastError = Result.NormalizationWarnings[0]; + ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError; + return Result; + } + + const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId; + Result = VisionClient->FinalizeVisionSession(SessionId); + if (Result.SessionId.IsEmpty()) + { + Result.SessionId = SessionId; + } + if (Result.FinalState.SourceSessionId.IsEmpty()) + { + Result.FinalState.SourceSessionId = SessionId; + } + if (Result.FinalSnapshot.State.SourceSessionId.IsEmpty()) + { + Result.FinalSnapshot.State.SourceSessionId = SessionId; + } + + TArray PendingRecognitionEvents; + for (const FString& SeedJson : Result.ReplaySeedEventsJson) + { + FHyperTwistReplayPacket SeedPacket; + if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedPacket)) + { + for (const FHyperTwistReplayEvent& Event : SeedPacket.Events) + { + if (HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType)) + { + PendingRecognitionEvents.Add(Event); + } + } + continue; + } + + FHyperTwistReplayEvent SeedEvent; + if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedEvent) + && HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(SeedEvent.EventType)) + { + PendingRecognitionEvents.Add(SeedEvent); + } + } + + const bool bHadProviderSeedEvents = PendingRecognitionEvents.Num() > 0; + const bool bHasProviderCorrectionEvent = PendingRecognitionEvents.ContainsByPredicate( + [](const FHyperTwistReplayEvent& Event) + { + return Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection; + } + ); + + if (Result.NormalizationWarnings.Num() > 0 && !bHasProviderCorrectionEvent) + { + FHyperTwistReplayRecognitionPayload CorrectionPayload = + HyperTwistTrainingSubsystemInternal::BuildFinalizeCorrectionPayload(Result); + CorrectionPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result); + + FString CorrectionPayloadJson; + if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CorrectionPayload, CorrectionPayloadJson)) + { + FHyperTwistReplayEvent CorrectionEvent; + CorrectionEvent.EventType = EHyperTwistReplayEventType::RecognitionCorrection; + CorrectionEvent.TimeMs = PendingRecognitionEvents.Num() > 0 + ? PendingRecognitionEvents.Last().TimeMs + : INDEX_NONE; + CorrectionEvent.PayloadJson = CorrectionPayloadJson; + + const int32 InsertIndex = PendingRecognitionEvents.IndexOfByPredicate( + [](const FHyperTwistReplayEvent& Event) + { + return Event.EventType == EHyperTwistReplayEventType::RecognitionCommit; + } + ); + if (InsertIndex == INDEX_NONE) + { + PendingRecognitionEvents.Add(CorrectionEvent); + } + else + { + PendingRecognitionEvents.Insert(CorrectionEvent, InsertIndex); + } + ++ActiveRecognitionSessionState.CorrectionEventCount; + } + } + + if (!bHadProviderSeedEvents) + { + FHyperTwistReplayRecognitionPayload CommitPayload = + HyperTwistTrainingSubsystemInternal::BuildFinalizeCommitPayload(Result); + CommitPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result); + + FString CommitPayloadJson; + if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CommitPayload, CommitPayloadJson)) + { + FHyperTwistReplayEvent CommitEvent; + CommitEvent.EventType = EHyperTwistReplayEventType::RecognitionCommit; + CommitEvent.PayloadJson = CommitPayloadJson; + PendingRecognitionEvents.Add(CommitEvent); + } + } + + bool bMutatedReplay = false; + for (const FHyperTwistReplayEvent& Event : PendingRecognitionEvents) + { + if (!HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType)) + { + continue; + } + + ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent( + ActiveRunState.ReplayPacket, + Event + ); + bMutatedReplay = true; + } + + FString CloseError; + if (!VisionClient->CloseVisionSession(SessionId, CloseError) && CloseError != TEXT("session-not-open")) + { + Result.NormalizationWarnings.AddUnique(CloseError); + } + + ActiveRecognitionSessionState.bSessionOpen = false; + ActiveRecognitionSessionState.bHasFinalizeResult = true; + ActiveRecognitionSessionState.LastFinalizeResult = Result; + ActiveRecognitionSessionState.ActiveSessionId.Reset(); + ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ++ActiveRecognitionSessionState.FinalizedSessionCount; + ActiveRecognitionSessionState.LastError = + Result.NormalizationWarnings.Num() > 0 ? FString::Join(Result.NormalizationWarnings, TEXT(" | ")) : FString(); + ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError; + + if (bMutatedReplay) + { + SynchronizeRecognitionReplayMutation(); + } + + return Result; +} + +bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutError) +{ + OutError.Reset(); + + if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty()) + { + RefreshRecognitionServiceHealth(); + return true; + } + + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + if (VisionClient == nullptr) + { + OutError = TEXT("recognition-client-unavailable"); + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId; + const bool bClosed = VisionClient->CloseVisionSession(SessionId, OutError); + if (!bClosed && OutError != TEXT("session-not-open")) + { + ActiveRecognitionSessionState.LastError = OutError; + RefreshRecognitionServiceHealth(); + return false; + } + + ActiveRecognitionSessionState.bSessionOpen = false; + ActiveRecognitionSessionState.ActiveSessionId.Reset(); + ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveRecognitionSessionState.LastError.Reset(); + RefreshRecognitionServiceHealth(); + return true; +} + void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun() { ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState(); @@ -3814,6 +4424,154 @@ void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun() bHasActiveMethodDrillRun = false; } +UObject* UHyperTwistTrainingSubsystem::ResolveRecognitionVisionClientObject() +{ + LoadConfig(); + + const bool bUseMockClient = RecognitionClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase); + const bool bNeedsNewClient = ActiveRecognitionVisionClientObject == nullptr + || (bUseMockClient && !ActiveRecognitionVisionClientObject->IsA()) + || (!bUseMockClient && !ActiveRecognitionVisionClientObject->IsA()); + if (bNeedsNewClient) + { + if (bUseMockClient) + { + ActiveRecognitionVisionClientObject = NewObject(this); + } + else + { + UHyperTwistHttpVisionClient* HttpVisionClient = NewObject(this); + HttpVisionClient->LoadConfig(); + ActiveRecognitionVisionClientObject = HttpVisionClient; + } + } + + return ActiveRecognitionVisionClientObject; +} + +FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognitionSessionConfig() const +{ + FHyperTwistVisionSessionConfig SessionConfig = ActiveRecognitionSessionState.SessionConfig; + if (!HasActiveRun()) + { + return SessionConfig; + } + + const FHyperTwistPuzzleDefinitionRef& ReplayDefinition = ActiveRunState.ReplayPacket.PuzzleDefinition; + const FHyperTwistTrainingCase* ActiveCase = ActiveRunState.CurrentSelection.IsStructurallyValid() + ? &ActiveRunState.CurrentSelection.TrainingCase + : HyperTwistTrainingSubsystemInternal::FindFirstStructurallyValidCase(ActiveRunState.ActiveDeck); + + SessionConfig.SessionId = !ActiveRecognitionSessionState.ActiveSessionId.IsEmpty() + ? ActiveRecognitionSessionState.ActiveSessionId + : HyperTwistTrainingSubsystemInternal::BuildRecognitionSessionId( + ActiveRunState.Session, + ActiveRecognitionSessionState.FinalizedSessionCount + 1 + ); + SessionConfig.PuzzleId = ReplayDefinition.IsStructurallyValid() + ? ReplayDefinition.PuzzleId + : (ActiveCase != nullptr && !ActiveCase->PuzzleId.IsEmpty() ? ActiveCase->PuzzleId : TEXT("cube/3x3x3")); + SessionConfig.ExpectedPuzzleFamily = ReplayDefinition.IsStructurallyValid() + ? ReplayDefinition.PuzzleFamily + : HyperTwistTrainingSubsystemInternal::ResolvePuzzleFamilyFromPuzzleId(SessionConfig.PuzzleId); + SessionConfig.NotationProfile = ReplayDefinition.IsStructurallyValid() + ? ReplayDefinition.NotationProfile + : HyperTwistTrainingSubsystemInternal::ResolveNotationProfileFromPuzzleId(SessionConfig.PuzzleId); + SessionConfig.CaptureMode = EHyperTwistVisionCaptureMode::LiveCamera; + if (SessionConfig.FrameSize.X <= 0 || SessionConfig.FrameSize.Y <= 0) + { + SessionConfig.FrameSize = FIntPoint(1280, 720); + } + if (SessionConfig.MaxFrameRate <= 0) + { + SessionConfig.MaxFrameRate = 30; + } + + return SessionConfig; +} + +void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth() +{ + IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( + ResolveRecognitionVisionClientObject() + ); + ActiveRecognitionSessionState.bUsingProviderBackedClient = + ActiveRecognitionVisionClientObject != nullptr + && ActiveRecognitionVisionClientObject->IsA(); + if (VisionClient == nullptr) + { + ActiveRecognitionSessionState.ServiceHealth = FHyperTwistVisionServiceHealth(); + ActiveRecognitionSessionState.ServiceHealth.ProviderLabel = ActiveRecognitionSessionState.bUsingProviderBackedClient + ? TEXT("local-http-sidecar") + : TEXT("mock"); + ActiveRecognitionSessionState.ServiceHealth.ServiceEndpoint = ActiveRecognitionSessionState.bUsingProviderBackedClient + ? TEXT("http://127.0.0.1:8766") + : TEXT("in-process://mock"); + ActiveRecognitionSessionState.ServiceHealth.bReady = false; + ActiveRecognitionSessionState.ServiceHealth.LastError = TEXT("recognition-client-unavailable"); + return; + } + + ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth(); +} + +void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent( + const EHyperTwistReplayEventType EventType, + const FHyperTwistReplayRecognitionPayload& Payload, + const int32 PreferredTimeMs +) +{ + if (!HasActiveRun()) + { + return; + } + + FString PayloadJson; + if (!HyperTwistTrainingSubsystemInternal::SerializeStruct(Payload, PayloadJson)) + { + return; + } + + FHyperTwistReplayEvent ReplayEvent; + ReplayEvent.EventType = EventType; + ReplayEvent.PayloadJson = PayloadJson; + + const int32 LastTimeMs = ActiveRunState.ReplayPacket.Events.Num() > 0 + ? ActiveRunState.ReplayPacket.Events.Last().TimeMs + : 0; + const int32 BaseTimeMs = PreferredTimeMs >= 0 ? PreferredTimeMs : LastTimeMs + 1; + ReplayEvent.TimeMs = BaseTimeMs > LastTimeMs ? BaseTimeMs : LastTimeMs + 1; + ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent( + ActiveRunState.ReplayPacket, + ReplayEvent + ); +} + +void UHyperTwistTrainingSubsystem::SynchronizeRecognitionReplayMutation() +{ + if (ActiveRunState.ReplayPacket.Events.Num() > 0) + { + ActiveRunState.LastReplayEvent = ActiveRunState.ReplayPacket.Events.Last(); + } + else + { + ActiveRunState.LastReplayEvent = FHyperTwistReplayEvent(); + } + + RefreshSummary(); + TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState( + TrainingRepositoryState, + ActiveRunState + ); + RefreshRepositoryViews(); +} + +void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState() +{ + ActiveRecognitionSessionState = FHyperTwistTrainingRecognitionSessionState(); + RefreshRecognitionServiceHealth(); +} + void UHyperTwistTrainingSubsystem::RefreshSummary() { if (!ActiveRunState.Session.IsStructurallyValid()) diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h index df2bf6e..b131ffe 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h @@ -21,6 +21,7 @@ public: virtual FHyperTwistVisionPreviewResult SubmitVisionFrame(const FHyperTwistVisionFrameEnvelope& Frame) = 0; virtual FHyperTwistVisionCommitResult CommitVisionObservation(const FHyperTwistVisionCommitRequest& Request) = 0; virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) = 0; + virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) = 0; virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const = 0; }; @@ -42,6 +43,9 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override; + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override; + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override; @@ -77,6 +81,9 @@ public: UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") FString FinalizeSessionPath = TEXT("/vision/session/finalize"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString CloseSessionPath = TEXT("/vision/session/close"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") FString HealthPath = TEXT("/vision/health"); @@ -98,6 +105,9 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override; + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override; + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h index c0e68aa..d25b9b3 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h @@ -53,6 +53,9 @@ public: UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training") FHyperTwistTrainingRunStepResult LastStepResult; + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition") + FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState; + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach") TArray CachedCoachSignals; @@ -136,6 +139,21 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training") void ClearActiveTrainingRun(); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool OpenActiveRecognitionSession(FString& OutError); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool CloseActiveRecognitionSession(FString& OutError); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Imported") FHyperTwistTrainingRunState ApplyActiveImportedRuntimeSelectorOption( const FString& SelectorId, diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h index 2afcfa0..42d9232 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h @@ -100,6 +100,27 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject")) static FHyperTwistTrainingSessionSummary GetActiveTrainingRunSummary(UObject* WorldContextObject); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static bool OpenActiveRecognitionSession(UObject* WorldContextObject, FString& OutError); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame( + UObject* WorldContextObject, + const FHyperTwistVisionFrameEnvelope& Frame + ); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static FHyperTwistVisionCommitResult CommitActiveRecognitionObservation( + UObject* WorldContextObject, + const FHyperTwistVisionCommitRequest& Request + ); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(UObject* WorldContextObject); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static bool CloseActiveRecognitionSession(UObject* WorldContextObject, FString& OutError); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Drill", meta = (WorldContext = "WorldContextObject")) static FHyperTwistTrainingMethodDrillSummary GetActiveMethodDrillSummary(UObject* WorldContextObject); @@ -173,6 +194,11 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject")) static FHyperTwistRecognitionReplaySummary GetActiveTrainingRecognitionReplaySummary(UObject* WorldContextObject); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject")) + static FHyperTwistTrainingRecognitionSessionState GetActiveTrainingRecognitionSessionState( + UObject* WorldContextObject + ); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach", meta = (WorldContext = "WorldContextObject")) static TArray GetActiveTrainingCoachSignals(UObject* WorldContextObject); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h index da9314a..55a3486 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h @@ -40,6 +40,9 @@ public: UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training") FHyperTwistTrainingRunStepResult LastStepResult; + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition") + FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState; + UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach") TArray CachedCoachSignals; @@ -117,6 +120,21 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training") void ClearRun(); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool OpenActiveRecognitionSession(FString& OutError); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool CloseActiveRecognitionSession(FString& OutError); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training") void RefreshCachedTrainingState(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h index b84619d..6f84002 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h @@ -8,7 +8,7 @@ #include "HyperTwistTraining/HyperTwistTrainingTypes.h" #include "HyperTwistTrainingSubsystem.generated.h" -UCLASS() +UCLASS(Config = Game, DefaultConfig) class UNREALHYPERTWIST_API UHyperTwistTrainingSubsystem : public UGameInstanceSubsystem { GENERATED_BODY() @@ -146,6 +146,9 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Training") FHyperTwistRecognitionReplaySummary GetActiveRecognitionReplaySummary() const; + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition") + FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const; + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach") TArray GetActiveCoachSignals() const; @@ -542,10 +545,35 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training") void ClearActiveRun(); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool OpenActiveRecognitionSession(FString& OutError); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") + bool CloseActiveRecognitionSession(FString& OutError); + private: void RefreshSummary(); void RefreshMethodDrillSummary(); void RefreshRepositoryViews(); + FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const; + UObject* ResolveRecognitionVisionClientObject(); + void RefreshRecognitionServiceHealth(); + void AppendRecognitionReplayEvent( + EHyperTwistReplayEventType EventType, + const FHyperTwistReplayRecognitionPayload& Payload, + int32 PreferredTimeMs + ); + void SynchronizeRecognitionReplayMutation(); + void ResetRecognitionSessionState(); TArray BuildTrainingSessionTemplatesForUser( const FString& UserId, const FString& ReferenceUtc @@ -611,6 +639,9 @@ private: UPROPERTY() FHyperTwistRecognitionReplaySummary ActiveRecognitionReplaySummary; + UPROPERTY() + FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState; + UPROPERTY() TArray ActiveCoachSignals; @@ -683,6 +714,12 @@ private: UPROPERTY() FHyperTwistTrainingReviewProgramSummary ActiveReviewProgramSummary; + UPROPERTY(Config) + FString RecognitionClientKind = TEXT("http-sidecar"); + + UPROPERTY() + TObjectPtr ActiveRecognitionVisionClientObject; + UPROPERTY() bool bHasActiveRun = false; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h index d6a2906..cc3f377 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h @@ -2,6 +2,7 @@ #include "CoreMinimal.h" #include "HyperTwistCore/HyperTwistCoreTypes.h" +#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h" #include "HyperTwistReplay/HyperTwistReplayTypes.h" #include "HyperTwistTrainingTypes.generated.h" @@ -1077,6 +1078,66 @@ struct FHyperTwistTrainingRunState } }; +USTRUCT(BlueprintType) +struct FHyperTwistTrainingRecognitionSessionState +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSessionOpen = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bUsingProviderBackedClient = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bHasPreviewResult = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bHasCommitResult = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bHasFinalizeResult = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ActiveSessionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString OpenedAtUtc; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastUpdatedAtUtc; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastError; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SubmittedFrameCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 CommittedObservationCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 CorrectionEventCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 FinalizedSessionCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistVisionSessionConfig SessionConfig; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistVisionServiceHealth ServiceHealth; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistVisionPreviewResult LastPreviewResult; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistVisionCommitResult LastCommitResult; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistVisionFinalizeResult LastFinalizeResult; +}; + USTRUCT(BlueprintType) struct FHyperTwistTrainingRunStepResult { diff --git a/docs/arch/HYPERTWIST_PHASE4_RECOGNITION_RUNTIME_HANDOFF_PACKET_2026-05-06.md b/docs/arch/HYPERTWIST_PHASE4_RECOGNITION_RUNTIME_HANDOFF_PACKET_2026-05-06.md new file mode 100644 index 0000000..7464356 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE4_RECOGNITION_RUNTIME_HANDOFF_PACKET_2026-05-06.md @@ -0,0 +1,127 @@ +# HyperTwist Phase 4 recognition runtime handoff packet + +Created on `2026-05-06` + +Status: + +- first-party HyperTwist packet +- bounded Phase `4` recognition runtime-handoff slice + +## Purpose + +This packet closes the first honest runtime handoff after the provider-backed recognition client landed. + +The open task is: + +- move recognition from a sidecar contract plus standalone client into owned training runtime state, replay state, and coaching refresh flow + +It is not: + +- a large recognition UI pass +- a websocket/live-stream rewrite +- a speech or simulation packet +- a broad refactor of the training subsystem + +## Scope + +Bounded lane: + +- complete recognition session lifecycle ownership in `HyperTwistTraining` +- expose runtime open / preview / commit / finalize / close surfaces through the subsystem, runtime library, panel widget, and session actor +- append normalized recognition replay events into the active run packet +- refresh repository/coaching derived state after replay mutation so existing replay-to-coach logic can wake up on live recognition data + +Out of scope: + +- broad dashboard redesign +- provider-specific protocol expansion beyond the current HTTP sidecar contract +- new gameplay content or speech flows +- exact puzzle-state engine deepening + +## Why this was the right next packet + +Before this slice: + +- the repo had a provider-backed recognition client +- replay review analytics and recognition replay summaries already existed +- coach-signal derivation already reacted to recognition replay ambiguity and correction pressure + +But recognition still did not flow through the owned training runtime. + +So the missing bridge was: + +- own the recognition session in `HyperTwistTraining` +- append replay events from recognition runtime activity +- let the existing replay/training/coaching stack consume those events without inventing a second parallel lane + +## What landed + +Primary code changes: + +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h` + - added explicit `CloseVisionSession(...)` to the owned vision-client contract +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp` +- `UnrealHyperTwist/Config/DefaultGame.ini` + - completed close-session support for both mock and provider-backed client implementations +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h` + - added `FHyperTwistTrainingRecognitionSessionState` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp` + - added owned runtime methods for: + - open active recognition session + - submit active recognition frame + - commit active recognition observation + - finalize active recognition session + - close active recognition session + - added owned recognition session state, provider-health refresh, replay-event append, and replay-mutation synchronization helpers + - recognition replay mutation now records run state and refreshes repository/coaching views in the same subsystem pass +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp` + - exposed the new recognition runtime lane through the existing world-context runtime surface +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp` + - added thin runtime-consumer wrappers and cached recognition session state + +## Runtime behavior now + +The active training runtime now owns: + +- recognition session open/closed state +- provider-health inspection state +- last preview / commit / finalize results +- cumulative frame / commit / correction / finalize counters + +Recognition runtime activity now mutates the active replay packet through owned event types: + +- first frame -> `RecognitionPreview` +- later frames -> `RecognitionScan` +- commit warnings / missing units -> `RecognitionCorrection` +- commit results -> `RecognitionCommit` +- finalize outputs -> replay-seed recognition events when provided, otherwise a synthesized final commit + +After replay mutation: + +- the active run summary is refreshed +- run state is persisted back into the training repository +- repository-derived replay review analytics, recognition replay summaries, and coach signals are rebuilt in the existing same-pass flow + +## Acceptance criteria + +- recognition session lifecycle is now owned by first-party training runtime code +- provider-backed and mock clients both satisfy the full open/submit/commit/finalize/close contract +- recognition runtime events reach the active replay packet +- replay mutation wakes up the existing repository/coaching refresh path +- the panel widget and session actor can consume the runtime lane without bypassing the subsystem +- full product build succeeds + +## Validation checklist + +1. build `UnrealHyperTwist.sln` / `UnrealHyperTwistEditor` +2. confirm the subsystem, runtime library, panel widget, and session actor all compile together +3. confirm no Phase `3` generated-mode packet was reopened +4. confirm the provider-backed client lane remains bounded to the owned vision contract + +That is the packet.