Wire recognition runtime into replay and coaching flow
This commit is contained in:
parent
1860f1488e
commit
44ecb902a0
14 changed files with 1261 additions and 4 deletions
|
|
@ -12,5 +12,6 @@ OpenSessionPath=/vision/session/open
|
|||
SubmitFramePath=/vision/frame
|
||||
CommitObservationPath=/vision/commit
|
||||
FinalizeSessionPath=/vision/session/finalize
|
||||
CloseSessionPath=/vision/session/close
|
||||
HealthPath=/vision/health
|
||||
RequestTimeoutSeconds=5.0
|
||||
|
|
|
|||
|
|
@ -343,6 +343,27 @@ FHyperTwistVisionFinalizeResult UHyperTwistHttpVisionClient::FinalizeVisionSessi
|
|||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistHttpVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
|
||||
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
|
||||
*this,
|
||||
TEXT("POST"),
|
||||
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath),
|
||||
HyperTwistHttpVisionClientInternal::MakeSessionIdJson(SessionId),
|
||||
Response))
|
||||
{
|
||||
OutError = Response.Error;
|
||||
LastTransportError = OutError;
|
||||
return false;
|
||||
}
|
||||
|
||||
LastTransportError.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistVisionServiceHealth UHyperTwistHttpVisionClient::GetVisionServiceHealth() const
|
||||
{
|
||||
FHyperTwistVisionServiceHealth Health;
|
||||
|
|
|
|||
|
|
@ -68,6 +68,21 @@ FHyperTwistVisionFinalizeResult UHyperTwistMockVisionClient::FinalizeVisionSessi
|
|||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistMockVisionClient::CloseVisionSession(const FString& SessionId, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!SessionConfigs.Contains(SessionId))
|
||||
{
|
||||
OutError = TEXT("session-not-open");
|
||||
return false;
|
||||
}
|
||||
|
||||
SessionConfigs.Remove(SessionId);
|
||||
SessionFrameCounts.Remove(SessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistVisionServiceHealth UHyperTwistMockVisionClient::GetVisionServiceHealth() const
|
||||
{
|
||||
FHyperTwistVisionServiceHealth Health = UHyperTwistContractLibrary::MakeMockVisionServiceHealth();
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ void UHyperTwistTrainingPanelWidget::RefreshTrainingState()
|
|||
{
|
||||
CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this);
|
||||
CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this);
|
||||
CachedRecognitionSessionState =
|
||||
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this);
|
||||
UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState(
|
||||
this,
|
||||
CachedImportedRuntimeSelectionState
|
||||
|
|
@ -129,6 +131,48 @@ void UHyperTwistTrainingPanelWidget::ClearActiveTrainingRun()
|
|||
RefreshTrainingState();
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingPanelWidget::OpenActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError);
|
||||
RefreshTrainingState();
|
||||
return bOpened;
|
||||
}
|
||||
|
||||
FHyperTwistVisionPreviewResult UHyperTwistTrainingPanelWidget::SubmitActiveRecognitionFrame(
|
||||
const FHyperTwistVisionFrameEnvelope& Frame
|
||||
)
|
||||
{
|
||||
const FHyperTwistVisionPreviewResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame);
|
||||
RefreshTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCommitResult UHyperTwistTrainingPanelWidget::CommitActiveRecognitionObservation(
|
||||
const FHyperTwistVisionCommitRequest& Request
|
||||
)
|
||||
{
|
||||
const FHyperTwistVisionCommitResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request);
|
||||
RefreshTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionFinalizeResult UHyperTwistTrainingPanelWidget::FinalizeActiveRecognitionSession()
|
||||
{
|
||||
const FHyperTwistVisionFinalizeResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this);
|
||||
RefreshTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingPanelWidget::CloseActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError);
|
||||
RefreshTrainingState();
|
||||
return bClosed;
|
||||
}
|
||||
|
||||
FHyperTwistTrainingRunState UHyperTwistTrainingPanelWidget::ApplyActiveImportedRuntimeSelectorOption(
|
||||
const FString& SelectorId,
|
||||
const FString& OptionId
|
||||
|
|
|
|||
|
|
@ -231,6 +231,72 @@ FHyperTwistTrainingSessionSummary UHyperTwistTrainingRuntimeLibrary::GetActiveTr
|
|||
return FHyperTwistTrainingSessionSummary();
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(
|
||||
UObject* WorldContextObject,
|
||||
FString& OutError
|
||||
)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->OpenActiveRecognitionSession(OutError);
|
||||
}
|
||||
|
||||
OutError = TEXT("Training subsystem was unavailable.");
|
||||
return false;
|
||||
}
|
||||
|
||||
FHyperTwistVisionPreviewResult UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(
|
||||
UObject* WorldContextObject,
|
||||
const FHyperTwistVisionFrameEnvelope& Frame
|
||||
)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->SubmitActiveRecognitionFrame(Frame);
|
||||
}
|
||||
|
||||
return FHyperTwistVisionPreviewResult();
|
||||
}
|
||||
|
||||
FHyperTwistVisionCommitResult UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(
|
||||
UObject* WorldContextObject,
|
||||
const FHyperTwistVisionCommitRequest& Request
|
||||
)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->CommitActiveRecognitionObservation(Request);
|
||||
}
|
||||
|
||||
return FHyperTwistVisionCommitResult();
|
||||
}
|
||||
|
||||
FHyperTwistVisionFinalizeResult UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(
|
||||
UObject* WorldContextObject
|
||||
)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->FinalizeActiveRecognitionSession();
|
||||
}
|
||||
|
||||
return FHyperTwistVisionFinalizeResult();
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(
|
||||
UObject* WorldContextObject,
|
||||
FString& OutError
|
||||
)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->CloseActiveRecognitionSession(OutError);
|
||||
}
|
||||
|
||||
OutError = TEXT("Training subsystem was unavailable.");
|
||||
return false;
|
||||
}
|
||||
|
||||
FHyperTwistTrainingMethodDrillSummary UHyperTwistTrainingRuntimeLibrary::GetActiveMethodDrillSummary(
|
||||
UObject* WorldContextObject
|
||||
)
|
||||
|
|
@ -450,6 +516,17 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingRuntimeLibrary::GetActive
|
|||
return FHyperTwistRecognitionReplaySummary();
|
||||
}
|
||||
|
||||
FHyperTwistTrainingRecognitionSessionState
|
||||
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(UObject* WorldContextObject)
|
||||
{
|
||||
if (UHyperTwistTrainingSubsystem* TrainingSubsystem = GetTrainingSubsystem(WorldContextObject))
|
||||
{
|
||||
return TrainingSubsystem->GetActiveRecognitionSessionState();
|
||||
}
|
||||
|
||||
return FHyperTwistTrainingRecognitionSessionState();
|
||||
}
|
||||
|
||||
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingCoachSignals(
|
||||
UObject* WorldContextObject
|
||||
)
|
||||
|
|
|
|||
|
|
@ -81,10 +81,54 @@ void AHyperTwistTrainingSessionActor::ClearRun()
|
|||
RefreshCachedTrainingState();
|
||||
}
|
||||
|
||||
bool AHyperTwistTrainingSessionActor::OpenActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
const bool bOpened = UHyperTwistTrainingRuntimeLibrary::OpenActiveRecognitionSession(this, OutError);
|
||||
RefreshCachedTrainingState();
|
||||
return bOpened;
|
||||
}
|
||||
|
||||
FHyperTwistVisionPreviewResult AHyperTwistTrainingSessionActor::SubmitActiveRecognitionFrame(
|
||||
const FHyperTwistVisionFrameEnvelope& Frame
|
||||
)
|
||||
{
|
||||
const FHyperTwistVisionPreviewResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::SubmitActiveRecognitionFrame(this, Frame);
|
||||
RefreshCachedTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCommitResult AHyperTwistTrainingSessionActor::CommitActiveRecognitionObservation(
|
||||
const FHyperTwistVisionCommitRequest& Request
|
||||
)
|
||||
{
|
||||
const FHyperTwistVisionCommitResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::CommitActiveRecognitionObservation(this, Request);
|
||||
RefreshCachedTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionFinalizeResult AHyperTwistTrainingSessionActor::FinalizeActiveRecognitionSession()
|
||||
{
|
||||
const FHyperTwistVisionFinalizeResult Result =
|
||||
UHyperTwistTrainingRuntimeLibrary::FinalizeActiveRecognitionSession(this);
|
||||
RefreshCachedTrainingState();
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool AHyperTwistTrainingSessionActor::CloseActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
const bool bClosed = UHyperTwistTrainingRuntimeLibrary::CloseActiveRecognitionSession(this, OutError);
|
||||
RefreshCachedTrainingState();
|
||||
return bClosed;
|
||||
}
|
||||
|
||||
void AHyperTwistTrainingSessionActor::RefreshCachedTrainingState()
|
||||
{
|
||||
CachedRunState = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunState(this);
|
||||
CachedRunSummary = UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRunSummary(this);
|
||||
CachedRecognitionSessionState =
|
||||
UHyperTwistTrainingRuntimeLibrary::GetActiveTrainingRecognitionSessionState(this);
|
||||
UHyperTwistTrainingRuntimeLibrary::TryGetActiveImportedRuntimeSelectionState(
|
||||
this,
|
||||
CachedImportedRuntimeSelectionState
|
||||
|
|
|
|||
|
|
@ -1,12 +1,15 @@
|
|||
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
|
||||
|
||||
#include "HyperTwistRecognition/HyperTwistVisionClient.h"
|
||||
#include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h"
|
||||
#include "HyperTwistReplay/HyperTwistReplayLibrary.h"
|
||||
#include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingCatalogLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingCoachLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h"
|
||||
#include "JsonObjectConverter.h"
|
||||
|
||||
namespace HyperTwistTrainingSubsystemInternal
|
||||
{
|
||||
|
|
@ -1144,6 +1147,119 @@ namespace HyperTwistTrainingSubsystemInternal
|
|||
QueueEntry.SuggestedMode == EHyperTwistTrainingDeliveryMode::RecognitionAssisted;
|
||||
return Brief;
|
||||
}
|
||||
|
||||
template <typename TStruct>
|
||||
bool SerializeStruct(const TStruct& Value, FString& OutJson)
|
||||
{
|
||||
return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0);
|
||||
}
|
||||
|
||||
template <typename TStruct>
|
||||
bool DeserializeStruct(const FString& Json, TStruct& OutValue)
|
||||
{
|
||||
return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0);
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* ResolveVisionClient(UObject* Candidate)
|
||||
{
|
||||
return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistVisionClient::StaticClass())
|
||||
? Cast<IHyperTwistVisionClient>(Candidate)
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
|
||||
{
|
||||
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
|
||||
return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
|
||||
}
|
||||
|
||||
FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal)
|
||||
{
|
||||
return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal));
|
||||
}
|
||||
|
||||
EHyperTwistPuzzleFamily ResolvePuzzleFamilyFromPuzzleId(const FString& PuzzleId)
|
||||
{
|
||||
if (PuzzleId.StartsWith(TEXT("hypercube/")))
|
||||
{
|
||||
return EHyperTwistPuzzleFamily::Hypercube;
|
||||
}
|
||||
if (PuzzleId.StartsWith(TEXT("cube/")))
|
||||
{
|
||||
return EHyperTwistPuzzleFamily::ClassicCube;
|
||||
}
|
||||
return EHyperTwistPuzzleFamily::Other;
|
||||
}
|
||||
|
||||
FString ResolveNotationProfileFromPuzzleId(const FString& PuzzleId)
|
||||
{
|
||||
if (PuzzleId.StartsWith(TEXT("hypercube/")))
|
||||
{
|
||||
return TEXT("hyper-4d");
|
||||
}
|
||||
return TEXT("classic-wca");
|
||||
}
|
||||
|
||||
bool IsRecognitionReplayEventType(const EHyperTwistReplayEventType EventType)
|
||||
{
|
||||
return EventType == EHyperTwistReplayEventType::RecognitionPreview
|
||||
|| EventType == EHyperTwistReplayEventType::RecognitionScan
|
||||
|| EventType == EHyperTwistReplayEventType::RecognitionCommit
|
||||
|| EventType == EHyperTwistReplayEventType::RecognitionCorrection;
|
||||
}
|
||||
|
||||
FHyperTwistReplayRecognitionPayload BuildFinalizeCorrectionPayload(
|
||||
const FHyperTwistVisionFinalizeResult& FinalizeResult
|
||||
)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload Payload;
|
||||
Payload.Snapshot = FinalizeResult.FinalSnapshot;
|
||||
Payload.Confidence = FinalizeResult.ConfidenceRollup;
|
||||
return Payload;
|
||||
}
|
||||
|
||||
FHyperTwistReplayRecognitionPayload BuildFinalizeCommitPayload(
|
||||
const FHyperTwistVisionFinalizeResult& FinalizeResult
|
||||
)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload Payload;
|
||||
Payload.Snapshot = FinalizeResult.FinalSnapshot;
|
||||
Payload.Confidence = FinalizeResult.ConfidenceRollup;
|
||||
return Payload;
|
||||
}
|
||||
|
||||
FHyperTwistStateSnapshot BuildSnapshotFromFinalState(
|
||||
const FHyperTwistVisionFinalizeResult& FinalizeResult
|
||||
)
|
||||
{
|
||||
FHyperTwistStateSnapshot Snapshot = FinalizeResult.FinalSnapshot;
|
||||
if (Snapshot.SnapshotId.IsEmpty())
|
||||
{
|
||||
Snapshot.SnapshotId = FString::Printf(TEXT("recognition_final_%s"), *FinalizeResult.SessionId);
|
||||
}
|
||||
if (!Snapshot.State.IsStructurallyValid() && FinalizeResult.FinalState.IsStructurallyValid())
|
||||
{
|
||||
Snapshot.State = FinalizeResult.FinalState;
|
||||
}
|
||||
if (Snapshot.DerivedHash.IsEmpty())
|
||||
{
|
||||
Snapshot.DerivedHash = FString::Printf(TEXT("recognition-final-%s"), *FinalizeResult.SessionId);
|
||||
}
|
||||
return Snapshot;
|
||||
}
|
||||
|
||||
int32 CountRecognitionCorrectionEvents(const TArray<FHyperTwistReplayEvent>& Events)
|
||||
{
|
||||
int32 Count = 0;
|
||||
for (const FHyperTwistReplayEvent& Event : Events)
|
||||
{
|
||||
if (Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection)
|
||||
{
|
||||
++Count;
|
||||
}
|
||||
}
|
||||
return Count;
|
||||
}
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::HasActiveRun() const
|
||||
|
|
@ -2064,6 +2180,11 @@ FHyperTwistRecognitionReplaySummary UHyperTwistTrainingSubsystem::GetActiveRecog
|
|||
return ActiveRecognitionReplaySummary;
|
||||
}
|
||||
|
||||
FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActiveRecognitionSessionState() const
|
||||
{
|
||||
return ActiveRecognitionSessionState;
|
||||
}
|
||||
|
||||
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const
|
||||
{
|
||||
return ActiveCoachSignals;
|
||||
|
|
@ -3477,13 +3598,19 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe
|
|||
EHyperTwistTrainingDeliveryMode Mode
|
||||
)
|
||||
{
|
||||
if (ActiveRecognitionSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
ResetRecognitionSessionState();
|
||||
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
|
||||
ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus();
|
||||
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
|
||||
ActiveCoachFollowUpBrief = FHyperTwistCoachBrief();
|
||||
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
|
||||
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
|
||||
ActiveRunState = UHyperTwistTrainingLibrary::StartTrainingRun(Deck, UserId, SessionId, Mode);
|
||||
ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest;
|
||||
ActiveRunState.ImportedGeneratedModeLaunchRequest = ActiveImportedGeneratedModeLaunchRequest;
|
||||
bHasActiveRun = ActiveRunState.IsStructurallyValid();
|
||||
RefreshSummary();
|
||||
TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState(
|
||||
|
|
@ -3736,6 +3863,12 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
|
|||
return FHyperTwistTrainingSession();
|
||||
}
|
||||
|
||||
if (ActiveRecognitionSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
|
||||
ActiveRunState.Session = UHyperTwistTrainingLibrary::CompleteTrainingSession(ActiveRunState.Session, bAbortSession);
|
||||
ActiveRunState.CurrentSelection = FHyperTwistTrainingCaseSelection();
|
||||
ActiveRunState.RemainingCaseIds.Reset();
|
||||
|
|
@ -3790,21 +3923,498 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
|
|||
);
|
||||
}
|
||||
}
|
||||
ResetRecognitionSessionState();
|
||||
RefreshRepositoryViews();
|
||||
return ActiveRunState.Session;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::ClearActiveRun()
|
||||
{
|
||||
if (ActiveRecognitionSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
ActiveRunState = FHyperTwistTrainingRunState();
|
||||
ActiveRunSummary = FHyperTwistTrainingSessionSummary();
|
||||
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
|
||||
ActiveImportedGeneratedModeLaunchRequest = FHyperTwistTrainingImportedGeneratedModeLaunchRequest();
|
||||
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
|
||||
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
|
||||
bHasActiveRun = false;
|
||||
ResetRecognitionSessionState();
|
||||
RefreshRepositoryViews();
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::OpenActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
OutError = TEXT("no-active-training-run");
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
OutError = TEXT("recognition-client-unavailable");
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FHyperTwistVisionSessionConfig SessionConfig = BuildActiveRecognitionSessionConfig();
|
||||
if (SessionConfig.SessionId.IsEmpty())
|
||||
{
|
||||
OutError = TEXT("recognition-session-id-unavailable");
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ActiveRecognitionSessionState.bSessionOpen
|
||||
&& ActiveRecognitionSessionState.ActiveSessionId == SessionConfig.SessionId)
|
||||
{
|
||||
ActiveRecognitionSessionState.SessionConfig = SessionConfig;
|
||||
ActiveRecognitionSessionState.LastError.Reset();
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ActiveRecognitionSessionState.bSessionOpen
|
||||
&& ActiveRecognitionSessionState.ActiveSessionId != SessionConfig.SessionId)
|
||||
{
|
||||
FString CloseError;
|
||||
VisionClient->CloseVisionSession(ActiveRecognitionSessionState.ActiveSessionId, CloseError);
|
||||
}
|
||||
|
||||
if (!VisionClient->OpenVisionSession(SessionConfig, OutError))
|
||||
{
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveRecognitionSessionState.bSessionOpen = true;
|
||||
ActiveRecognitionSessionState.bUsingProviderBackedClient =
|
||||
ActiveRecognitionVisionClientObject != nullptr
|
||||
&& ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>();
|
||||
ActiveRecognitionSessionState.ActiveSessionId = SessionConfig.SessionId;
|
||||
ActiveRecognitionSessionState.OpenedAtUtc = OpenedAtUtc;
|
||||
ActiveRecognitionSessionState.LastUpdatedAtUtc = OpenedAtUtc;
|
||||
ActiveRecognitionSessionState.LastError.Reset();
|
||||
ActiveRecognitionSessionState.SessionConfig = SessionConfig;
|
||||
ActiveRecognitionSessionState.bHasPreviewResult = false;
|
||||
ActiveRecognitionSessionState.bHasCommitResult = false;
|
||||
ActiveRecognitionSessionState.LastPreviewResult = FHyperTwistVisionPreviewResult();
|
||||
ActiveRecognitionSessionState.LastCommitResult = FHyperTwistVisionCommitResult();
|
||||
RefreshRecognitionServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistVisionPreviewResult UHyperTwistTrainingSubsystem::SubmitActiveRecognitionFrame(
|
||||
const FHyperTwistVisionFrameEnvelope& Frame
|
||||
)
|
||||
{
|
||||
FHyperTwistVisionPreviewResult Result;
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
Result.Guidance = TEXT("no-active-training-run");
|
||||
return Result;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
if (!OpenActiveRecognitionSession(OpenError))
|
||||
{
|
||||
Result.SessionId = !Frame.SessionId.IsEmpty() ? Frame.SessionId : BuildActiveRecognitionSessionConfig().SessionId;
|
||||
Result.FrameId = Frame.FrameId;
|
||||
Result.Guidance = OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError;
|
||||
return Result;
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
Result.Guidance = TEXT("recognition-client-unavailable");
|
||||
ActiveRecognitionSessionState.LastError = Result.Guidance;
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionFrameEnvelope NormalizedFrame = Frame;
|
||||
NormalizedFrame.SessionId = ActiveRecognitionSessionState.ActiveSessionId;
|
||||
if (NormalizedFrame.FrameOrdinal <= 0)
|
||||
{
|
||||
NormalizedFrame.FrameOrdinal = ActiveRecognitionSessionState.SubmittedFrameCount + 1;
|
||||
}
|
||||
if (NormalizedFrame.FrameId.IsEmpty())
|
||||
{
|
||||
NormalizedFrame.FrameId = HyperTwistTrainingSubsystemInternal::BuildRecognitionFrameId(
|
||||
NormalizedFrame.SessionId,
|
||||
NormalizedFrame.FrameOrdinal
|
||||
);
|
||||
}
|
||||
if (NormalizedFrame.CapturedAtUtc.IsEmpty())
|
||||
{
|
||||
NormalizedFrame.CapturedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
}
|
||||
|
||||
Result = VisionClient->SubmitVisionFrame(NormalizedFrame);
|
||||
if (Result.SessionId.IsEmpty())
|
||||
{
|
||||
Result.SessionId = NormalizedFrame.SessionId;
|
||||
}
|
||||
if (Result.FrameId.IsEmpty())
|
||||
{
|
||||
Result.FrameId = NormalizedFrame.FrameId;
|
||||
}
|
||||
if (Result.PreviewState.State.SourceSessionId.IsEmpty())
|
||||
{
|
||||
Result.PreviewState.State.SourceSessionId = NormalizedFrame.SessionId;
|
||||
}
|
||||
|
||||
ActiveRecognitionSessionState.bHasPreviewResult = true;
|
||||
ActiveRecognitionSessionState.LastPreviewResult = Result;
|
||||
ActiveRecognitionSessionState.SubmittedFrameCount = FMath::Max(
|
||||
ActiveRecognitionSessionState.SubmittedFrameCount,
|
||||
NormalizedFrame.FrameOrdinal
|
||||
);
|
||||
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveRecognitionSessionState.LastError.Reset();
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError.Reset();
|
||||
|
||||
FHyperTwistReplayRecognitionPayload Payload;
|
||||
Payload.Snapshot = Result.PreviewState;
|
||||
Payload.Confidence = Result.PreviewConfidence;
|
||||
Payload.ScanStage = NormalizedFrame.CameraPoseHint;
|
||||
AppendRecognitionReplayEvent(
|
||||
ActiveRecognitionSessionState.SubmittedFrameCount <= 1
|
||||
? EHyperTwistReplayEventType::RecognitionPreview
|
||||
: EHyperTwistReplayEventType::RecognitionScan,
|
||||
Payload,
|
||||
NormalizedFrame.SourceTimestampMs > 0 ? NormalizedFrame.SourceTimestampMs : INDEX_NONE
|
||||
);
|
||||
SynchronizeRecognitionReplayMutation();
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCommitResult UHyperTwistTrainingSubsystem::CommitActiveRecognitionObservation(
|
||||
const FHyperTwistVisionCommitRequest& Request
|
||||
)
|
||||
{
|
||||
FHyperTwistVisionCommitResult Result;
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
Result.Warnings.Add(TEXT("no-active-training-run"));
|
||||
return Result;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
if (!OpenActiveRecognitionSession(OpenError))
|
||||
{
|
||||
Result.SessionId = Request.SessionId;
|
||||
Result.CommitKind = Request.CommitKind;
|
||||
Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("recognition-session-open-failed") : OpenError);
|
||||
return Result;
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
Result.Warnings.Add(TEXT("recognition-client-unavailable"));
|
||||
ActiveRecognitionSessionState.LastError = Result.Warnings[0];
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCommitRequest NormalizedRequest = Request;
|
||||
NormalizedRequest.SessionId = ActiveRecognitionSessionState.ActiveSessionId;
|
||||
if (NormalizedRequest.CommitKind.IsEmpty())
|
||||
{
|
||||
NormalizedRequest.CommitKind = TEXT("face");
|
||||
}
|
||||
|
||||
Result = VisionClient->CommitVisionObservation(NormalizedRequest);
|
||||
if (Result.SessionId.IsEmpty())
|
||||
{
|
||||
Result.SessionId = NormalizedRequest.SessionId;
|
||||
}
|
||||
if (Result.CommitKind.IsEmpty())
|
||||
{
|
||||
Result.CommitKind = NormalizedRequest.CommitKind;
|
||||
}
|
||||
if (Result.Snapshot.State.SourceSessionId.IsEmpty())
|
||||
{
|
||||
Result.Snapshot.State.SourceSessionId = NormalizedRequest.SessionId;
|
||||
}
|
||||
|
||||
const FString ResolvedStage = !Result.CommittedUnit.IsEmpty()
|
||||
? Result.CommittedUnit
|
||||
: (!NormalizedRequest.TargetStage.IsEmpty() ? NormalizedRequest.TargetStage : NormalizedRequest.TargetFace);
|
||||
const bool bHasCorrectionSignals =
|
||||
Result.Warnings.Num() > 0 || Result.MissingUnits.Num() > 0;
|
||||
const bool bHasCommitSignal =
|
||||
!Result.CommittedUnit.IsEmpty()
|
||||
|| Result.Snapshot.IsStructurallyValid()
|
||||
|| Result.Snapshot.State.IsStructurallyValid()
|
||||
|| Result.Confidence > 0.0f;
|
||||
|
||||
if (bHasCorrectionSignals)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload CorrectionPayload;
|
||||
CorrectionPayload.Snapshot = Result.Snapshot;
|
||||
CorrectionPayload.Confidence = Result.Confidence;
|
||||
CorrectionPayload.CommittedFaceOrStage = ResolvedStage;
|
||||
AppendRecognitionReplayEvent(
|
||||
EHyperTwistReplayEventType::RecognitionCorrection,
|
||||
CorrectionPayload,
|
||||
INDEX_NONE
|
||||
);
|
||||
++ActiveRecognitionSessionState.CorrectionEventCount;
|
||||
}
|
||||
|
||||
if (bHasCommitSignal)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload CommitPayload;
|
||||
CommitPayload.Snapshot = Result.Snapshot;
|
||||
CommitPayload.Confidence = Result.Confidence;
|
||||
CommitPayload.CommittedFaceOrStage = ResolvedStage;
|
||||
AppendRecognitionReplayEvent(
|
||||
EHyperTwistReplayEventType::RecognitionCommit,
|
||||
CommitPayload,
|
||||
INDEX_NONE
|
||||
);
|
||||
++ActiveRecognitionSessionState.CommittedObservationCount;
|
||||
}
|
||||
|
||||
ActiveRecognitionSessionState.bHasCommitResult = true;
|
||||
ActiveRecognitionSessionState.LastCommitResult = Result;
|
||||
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveRecognitionSessionState.LastError =
|
||||
Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString();
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
|
||||
|
||||
if (bHasCorrectionSignals || bHasCommitSignal)
|
||||
{
|
||||
SynchronizeRecognitionReplayMutation();
|
||||
}
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistVisionFinalizeResult UHyperTwistTrainingSubsystem::FinalizeActiveRecognitionSession()
|
||||
{
|
||||
FHyperTwistVisionFinalizeResult Result;
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
Result.NormalizationWarnings.Add(TEXT("no-active-training-run"));
|
||||
return Result;
|
||||
}
|
||||
|
||||
if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty())
|
||||
{
|
||||
Result.NormalizationWarnings.Add(TEXT("recognition-session-not-open"));
|
||||
return Result;
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
Result.NormalizationWarnings.Add(TEXT("recognition-client-unavailable"));
|
||||
ActiveRecognitionSessionState.LastError = Result.NormalizationWarnings[0];
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
|
||||
return Result;
|
||||
}
|
||||
|
||||
const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId;
|
||||
Result = VisionClient->FinalizeVisionSession(SessionId);
|
||||
if (Result.SessionId.IsEmpty())
|
||||
{
|
||||
Result.SessionId = SessionId;
|
||||
}
|
||||
if (Result.FinalState.SourceSessionId.IsEmpty())
|
||||
{
|
||||
Result.FinalState.SourceSessionId = SessionId;
|
||||
}
|
||||
if (Result.FinalSnapshot.State.SourceSessionId.IsEmpty())
|
||||
{
|
||||
Result.FinalSnapshot.State.SourceSessionId = SessionId;
|
||||
}
|
||||
|
||||
TArray<FHyperTwistReplayEvent> PendingRecognitionEvents;
|
||||
for (const FString& SeedJson : Result.ReplaySeedEventsJson)
|
||||
{
|
||||
FHyperTwistReplayPacket SeedPacket;
|
||||
if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedPacket))
|
||||
{
|
||||
for (const FHyperTwistReplayEvent& Event : SeedPacket.Events)
|
||||
{
|
||||
if (HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType))
|
||||
{
|
||||
PendingRecognitionEvents.Add(Event);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
FHyperTwistReplayEvent SeedEvent;
|
||||
if (HyperTwistTrainingSubsystemInternal::DeserializeStruct(SeedJson, SeedEvent)
|
||||
&& HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(SeedEvent.EventType))
|
||||
{
|
||||
PendingRecognitionEvents.Add(SeedEvent);
|
||||
}
|
||||
}
|
||||
|
||||
const bool bHadProviderSeedEvents = PendingRecognitionEvents.Num() > 0;
|
||||
const bool bHasProviderCorrectionEvent = PendingRecognitionEvents.ContainsByPredicate(
|
||||
[](const FHyperTwistReplayEvent& Event)
|
||||
{
|
||||
return Event.EventType == EHyperTwistReplayEventType::RecognitionCorrection;
|
||||
}
|
||||
);
|
||||
|
||||
if (Result.NormalizationWarnings.Num() > 0 && !bHasProviderCorrectionEvent)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload CorrectionPayload =
|
||||
HyperTwistTrainingSubsystemInternal::BuildFinalizeCorrectionPayload(Result);
|
||||
CorrectionPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result);
|
||||
|
||||
FString CorrectionPayloadJson;
|
||||
if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CorrectionPayload, CorrectionPayloadJson))
|
||||
{
|
||||
FHyperTwistReplayEvent CorrectionEvent;
|
||||
CorrectionEvent.EventType = EHyperTwistReplayEventType::RecognitionCorrection;
|
||||
CorrectionEvent.TimeMs = PendingRecognitionEvents.Num() > 0
|
||||
? PendingRecognitionEvents.Last().TimeMs
|
||||
: INDEX_NONE;
|
||||
CorrectionEvent.PayloadJson = CorrectionPayloadJson;
|
||||
|
||||
const int32 InsertIndex = PendingRecognitionEvents.IndexOfByPredicate(
|
||||
[](const FHyperTwistReplayEvent& Event)
|
||||
{
|
||||
return Event.EventType == EHyperTwistReplayEventType::RecognitionCommit;
|
||||
}
|
||||
);
|
||||
if (InsertIndex == INDEX_NONE)
|
||||
{
|
||||
PendingRecognitionEvents.Add(CorrectionEvent);
|
||||
}
|
||||
else
|
||||
{
|
||||
PendingRecognitionEvents.Insert(CorrectionEvent, InsertIndex);
|
||||
}
|
||||
++ActiveRecognitionSessionState.CorrectionEventCount;
|
||||
}
|
||||
}
|
||||
|
||||
if (!bHadProviderSeedEvents)
|
||||
{
|
||||
FHyperTwistReplayRecognitionPayload CommitPayload =
|
||||
HyperTwistTrainingSubsystemInternal::BuildFinalizeCommitPayload(Result);
|
||||
CommitPayload.Snapshot = HyperTwistTrainingSubsystemInternal::BuildSnapshotFromFinalState(Result);
|
||||
|
||||
FString CommitPayloadJson;
|
||||
if (HyperTwistTrainingSubsystemInternal::SerializeStruct(CommitPayload, CommitPayloadJson))
|
||||
{
|
||||
FHyperTwistReplayEvent CommitEvent;
|
||||
CommitEvent.EventType = EHyperTwistReplayEventType::RecognitionCommit;
|
||||
CommitEvent.PayloadJson = CommitPayloadJson;
|
||||
PendingRecognitionEvents.Add(CommitEvent);
|
||||
}
|
||||
}
|
||||
|
||||
bool bMutatedReplay = false;
|
||||
for (const FHyperTwistReplayEvent& Event : PendingRecognitionEvents)
|
||||
{
|
||||
if (!HyperTwistTrainingSubsystemInternal::IsRecognitionReplayEventType(Event.EventType))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent(
|
||||
ActiveRunState.ReplayPacket,
|
||||
Event
|
||||
);
|
||||
bMutatedReplay = true;
|
||||
}
|
||||
|
||||
FString CloseError;
|
||||
if (!VisionClient->CloseVisionSession(SessionId, CloseError) && CloseError != TEXT("session-not-open"))
|
||||
{
|
||||
Result.NormalizationWarnings.AddUnique(CloseError);
|
||||
}
|
||||
|
||||
ActiveRecognitionSessionState.bSessionOpen = false;
|
||||
ActiveRecognitionSessionState.bHasFinalizeResult = true;
|
||||
ActiveRecognitionSessionState.LastFinalizeResult = Result;
|
||||
ActiveRecognitionSessionState.ActiveSessionId.Reset();
|
||||
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
++ActiveRecognitionSessionState.FinalizedSessionCount;
|
||||
ActiveRecognitionSessionState.LastError =
|
||||
Result.NormalizationWarnings.Num() > 0 ? FString::Join(Result.NormalizationWarnings, TEXT(" | ")) : FString();
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = ActiveRecognitionSessionState.LastError;
|
||||
|
||||
if (bMutatedReplay)
|
||||
{
|
||||
SynchronizeRecognitionReplayMutation();
|
||||
}
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!ActiveRecognitionSessionState.bSessionOpen || ActiveRecognitionSessionState.ActiveSessionId.IsEmpty())
|
||||
{
|
||||
RefreshRecognitionServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
OutError = TEXT("recognition-client-unavailable");
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FString SessionId = ActiveRecognitionSessionState.ActiveSessionId;
|
||||
const bool bClosed = VisionClient->CloseVisionSession(SessionId, OutError);
|
||||
if (!bClosed && OutError != TEXT("session-not-open"))
|
||||
{
|
||||
ActiveRecognitionSessionState.LastError = OutError;
|
||||
RefreshRecognitionServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
ActiveRecognitionSessionState.bSessionOpen = false;
|
||||
ActiveRecognitionSessionState.ActiveSessionId.Reset();
|
||||
ActiveRecognitionSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveRecognitionSessionState.LastError.Reset();
|
||||
RefreshRecognitionServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
|
||||
{
|
||||
ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState();
|
||||
|
|
@ -3814,6 +4424,154 @@ void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
|
|||
bHasActiveMethodDrillRun = false;
|
||||
}
|
||||
|
||||
UObject* UHyperTwistTrainingSubsystem::ResolveRecognitionVisionClientObject()
|
||||
{
|
||||
LoadConfig();
|
||||
|
||||
const bool bUseMockClient = RecognitionClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase);
|
||||
const bool bNeedsNewClient = ActiveRecognitionVisionClientObject == nullptr
|
||||
|| (bUseMockClient && !ActiveRecognitionVisionClientObject->IsA<UHyperTwistMockVisionClient>())
|
||||
|| (!bUseMockClient && !ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>());
|
||||
if (bNeedsNewClient)
|
||||
{
|
||||
if (bUseMockClient)
|
||||
{
|
||||
ActiveRecognitionVisionClientObject = NewObject<UHyperTwistMockVisionClient>(this);
|
||||
}
|
||||
else
|
||||
{
|
||||
UHyperTwistHttpVisionClient* HttpVisionClient = NewObject<UHyperTwistHttpVisionClient>(this);
|
||||
HttpVisionClient->LoadConfig();
|
||||
ActiveRecognitionVisionClientObject = HttpVisionClient;
|
||||
}
|
||||
}
|
||||
|
||||
return ActiveRecognitionVisionClientObject;
|
||||
}
|
||||
|
||||
FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognitionSessionConfig() const
|
||||
{
|
||||
FHyperTwistVisionSessionConfig SessionConfig = ActiveRecognitionSessionState.SessionConfig;
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
return SessionConfig;
|
||||
}
|
||||
|
||||
const FHyperTwistPuzzleDefinitionRef& ReplayDefinition = ActiveRunState.ReplayPacket.PuzzleDefinition;
|
||||
const FHyperTwistTrainingCase* ActiveCase = ActiveRunState.CurrentSelection.IsStructurallyValid()
|
||||
? &ActiveRunState.CurrentSelection.TrainingCase
|
||||
: HyperTwistTrainingSubsystemInternal::FindFirstStructurallyValidCase(ActiveRunState.ActiveDeck);
|
||||
|
||||
SessionConfig.SessionId = !ActiveRecognitionSessionState.ActiveSessionId.IsEmpty()
|
||||
? ActiveRecognitionSessionState.ActiveSessionId
|
||||
: HyperTwistTrainingSubsystemInternal::BuildRecognitionSessionId(
|
||||
ActiveRunState.Session,
|
||||
ActiveRecognitionSessionState.FinalizedSessionCount + 1
|
||||
);
|
||||
SessionConfig.PuzzleId = ReplayDefinition.IsStructurallyValid()
|
||||
? ReplayDefinition.PuzzleId
|
||||
: (ActiveCase != nullptr && !ActiveCase->PuzzleId.IsEmpty() ? ActiveCase->PuzzleId : TEXT("cube/3x3x3"));
|
||||
SessionConfig.ExpectedPuzzleFamily = ReplayDefinition.IsStructurallyValid()
|
||||
? ReplayDefinition.PuzzleFamily
|
||||
: HyperTwistTrainingSubsystemInternal::ResolvePuzzleFamilyFromPuzzleId(SessionConfig.PuzzleId);
|
||||
SessionConfig.NotationProfile = ReplayDefinition.IsStructurallyValid()
|
||||
? ReplayDefinition.NotationProfile
|
||||
: HyperTwistTrainingSubsystemInternal::ResolveNotationProfileFromPuzzleId(SessionConfig.PuzzleId);
|
||||
SessionConfig.CaptureMode = EHyperTwistVisionCaptureMode::LiveCamera;
|
||||
if (SessionConfig.FrameSize.X <= 0 || SessionConfig.FrameSize.Y <= 0)
|
||||
{
|
||||
SessionConfig.FrameSize = FIntPoint(1280, 720);
|
||||
}
|
||||
if (SessionConfig.MaxFrameRate <= 0)
|
||||
{
|
||||
SessionConfig.MaxFrameRate = 30;
|
||||
}
|
||||
|
||||
return SessionConfig;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
|
||||
{
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
ResolveRecognitionVisionClientObject()
|
||||
);
|
||||
ActiveRecognitionSessionState.bUsingProviderBackedClient =
|
||||
ActiveRecognitionVisionClientObject != nullptr
|
||||
&& ActiveRecognitionVisionClientObject->IsA<UHyperTwistHttpVisionClient>();
|
||||
if (VisionClient == nullptr)
|
||||
{
|
||||
ActiveRecognitionSessionState.ServiceHealth = FHyperTwistVisionServiceHealth();
|
||||
ActiveRecognitionSessionState.ServiceHealth.ProviderLabel = ActiveRecognitionSessionState.bUsingProviderBackedClient
|
||||
? TEXT("local-http-sidecar")
|
||||
: TEXT("mock");
|
||||
ActiveRecognitionSessionState.ServiceHealth.ServiceEndpoint = ActiveRecognitionSessionState.bUsingProviderBackedClient
|
||||
? TEXT("http://127.0.0.1:8766")
|
||||
: TEXT("in-process://mock");
|
||||
ActiveRecognitionSessionState.ServiceHealth.bReady = false;
|
||||
ActiveRecognitionSessionState.ServiceHealth.LastError = TEXT("recognition-client-unavailable");
|
||||
return;
|
||||
}
|
||||
|
||||
ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth();
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent(
|
||||
const EHyperTwistReplayEventType EventType,
|
||||
const FHyperTwistReplayRecognitionPayload& Payload,
|
||||
const int32 PreferredTimeMs
|
||||
)
|
||||
{
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
FString PayloadJson;
|
||||
if (!HyperTwistTrainingSubsystemInternal::SerializeStruct(Payload, PayloadJson))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
FHyperTwistReplayEvent ReplayEvent;
|
||||
ReplayEvent.EventType = EventType;
|
||||
ReplayEvent.PayloadJson = PayloadJson;
|
||||
|
||||
const int32 LastTimeMs = ActiveRunState.ReplayPacket.Events.Num() > 0
|
||||
? ActiveRunState.ReplayPacket.Events.Last().TimeMs
|
||||
: 0;
|
||||
const int32 BaseTimeMs = PreferredTimeMs >= 0 ? PreferredTimeMs : LastTimeMs + 1;
|
||||
ReplayEvent.TimeMs = BaseTimeMs > LastTimeMs ? BaseTimeMs : LastTimeMs + 1;
|
||||
ActiveRunState.ReplayPacket = UHyperTwistReplayLibrary::AppendReplayEvent(
|
||||
ActiveRunState.ReplayPacket,
|
||||
ReplayEvent
|
||||
);
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::SynchronizeRecognitionReplayMutation()
|
||||
{
|
||||
if (ActiveRunState.ReplayPacket.Events.Num() > 0)
|
||||
{
|
||||
ActiveRunState.LastReplayEvent = ActiveRunState.ReplayPacket.Events.Last();
|
||||
}
|
||||
else
|
||||
{
|
||||
ActiveRunState.LastReplayEvent = FHyperTwistReplayEvent();
|
||||
}
|
||||
|
||||
RefreshSummary();
|
||||
TrainingRepositoryState = UHyperTwistTrainingRepositoryLibrary::RecordRunState(
|
||||
TrainingRepositoryState,
|
||||
ActiveRunState
|
||||
);
|
||||
RefreshRepositoryViews();
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState()
|
||||
{
|
||||
ActiveRecognitionSessionState = FHyperTwistTrainingRecognitionSessionState();
|
||||
RefreshRecognitionServiceHealth();
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::RefreshSummary()
|
||||
{
|
||||
if (!ActiveRunState.Session.IsStructurallyValid())
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ public:
|
|||
virtual FHyperTwistVisionPreviewResult SubmitVisionFrame(const FHyperTwistVisionFrameEnvelope& Frame) = 0;
|
||||
virtual FHyperTwistVisionCommitResult CommitVisionObservation(const FHyperTwistVisionCommitRequest& Request) = 0;
|
||||
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) = 0;
|
||||
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) = 0;
|
||||
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const = 0;
|
||||
};
|
||||
|
||||
|
|
@ -42,6 +43,9 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;
|
||||
|
||||
|
|
@ -77,6 +81,9 @@ public:
|
|||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString FinalizeSessionPath = TEXT("/vision/session/finalize");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString CloseSessionPath = TEXT("/vision/session/close");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString HealthPath = TEXT("/vision/health");
|
||||
|
||||
|
|
@ -98,6 +105,9 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool CloseVisionSession(const FString& SessionId, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,9 @@ public:
|
|||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training")
|
||||
FHyperTwistTrainingRunStepResult LastStepResult;
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState;
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach")
|
||||
TArray<FHyperTwistCoachSignal> CachedCoachSignals;
|
||||
|
||||
|
|
@ -136,6 +139,21 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
|
||||
void ClearActiveTrainingRun();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool OpenActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool CloseActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Imported")
|
||||
FHyperTwistTrainingRunState ApplyActiveImportedRuntimeSelectorOption(
|
||||
const FString& SelectorId,
|
||||
|
|
|
|||
|
|
@ -100,6 +100,27 @@ public:
|
|||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistTrainingSessionSummary GetActiveTrainingRunSummary(UObject* WorldContextObject);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static bool OpenActiveRecognitionSession(UObject* WorldContextObject, FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(
|
||||
UObject* WorldContextObject,
|
||||
const FHyperTwistVisionFrameEnvelope& Frame
|
||||
);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(
|
||||
UObject* WorldContextObject,
|
||||
const FHyperTwistVisionCommitRequest& Request
|
||||
);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession(UObject* WorldContextObject);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static bool CloseActiveRecognitionSession(UObject* WorldContextObject, FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Drill", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistTrainingMethodDrillSummary GetActiveMethodDrillSummary(UObject* WorldContextObject);
|
||||
|
||||
|
|
@ -173,6 +194,11 @@ public:
|
|||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistRecognitionReplaySummary GetActiveTrainingRecognitionReplaySummary(UObject* WorldContextObject);
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition", meta = (WorldContext = "WorldContextObject"))
|
||||
static FHyperTwistTrainingRecognitionSessionState GetActiveTrainingRecognitionSessionState(
|
||||
UObject* WorldContextObject
|
||||
);
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach", meta = (WorldContext = "WorldContextObject"))
|
||||
static TArray<FHyperTwistCoachSignal> GetActiveTrainingCoachSignals(UObject* WorldContextObject);
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ public:
|
|||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training")
|
||||
FHyperTwistTrainingRunStepResult LastStepResult;
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistTrainingRecognitionSessionState CachedRecognitionSessionState;
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly, Category = "HyperTwist|Training|Coach")
|
||||
TArray<FHyperTwistCoachSignal> CachedCoachSignals;
|
||||
|
||||
|
|
@ -117,6 +120,21 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
|
||||
void ClearRun();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool OpenActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool CloseActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
|
||||
void RefreshCachedTrainingState();
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
#include "HyperTwistTraining/HyperTwistTrainingTypes.h"
|
||||
#include "HyperTwistTrainingSubsystem.generated.h"
|
||||
|
||||
UCLASS()
|
||||
UCLASS(Config = Game, DefaultConfig)
|
||||
class UNREALHYPERTWIST_API UHyperTwistTrainingSubsystem : public UGameInstanceSubsystem
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
|
@ -146,6 +146,9 @@ public:
|
|||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training")
|
||||
FHyperTwistRecognitionReplaySummary GetActiveRecognitionReplaySummary() const;
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const;
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach")
|
||||
TArray<FHyperTwistCoachSignal> GetActiveCoachSignals() const;
|
||||
|
||||
|
|
@ -542,10 +545,35 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training")
|
||||
void ClearActiveRun();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool OpenActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionPreviewResult SubmitActiveRecognitionFrame(const FHyperTwistVisionFrameEnvelope& Frame);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionCommitResult CommitActiveRecognitionObservation(const FHyperTwistVisionCommitRequest& Request);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistVisionFinalizeResult FinalizeActiveRecognitionSession();
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool CloseActiveRecognitionSession(FString& OutError);
|
||||
|
||||
private:
|
||||
void RefreshSummary();
|
||||
void RefreshMethodDrillSummary();
|
||||
void RefreshRepositoryViews();
|
||||
FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const;
|
||||
UObject* ResolveRecognitionVisionClientObject();
|
||||
void RefreshRecognitionServiceHealth();
|
||||
void AppendRecognitionReplayEvent(
|
||||
EHyperTwistReplayEventType EventType,
|
||||
const FHyperTwistReplayRecognitionPayload& Payload,
|
||||
int32 PreferredTimeMs
|
||||
);
|
||||
void SynchronizeRecognitionReplayMutation();
|
||||
void ResetRecognitionSessionState();
|
||||
TArray<FHyperTwistTrainingSessionTemplate> BuildTrainingSessionTemplatesForUser(
|
||||
const FString& UserId,
|
||||
const FString& ReferenceUtc
|
||||
|
|
@ -611,6 +639,9 @@ private:
|
|||
UPROPERTY()
|
||||
FHyperTwistRecognitionReplaySummary ActiveRecognitionReplaySummary;
|
||||
|
||||
UPROPERTY()
|
||||
FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState;
|
||||
|
||||
UPROPERTY()
|
||||
TArray<FHyperTwistCoachSignal> ActiveCoachSignals;
|
||||
|
||||
|
|
@ -683,6 +714,12 @@ private:
|
|||
UPROPERTY()
|
||||
FHyperTwistTrainingReviewProgramSummary ActiveReviewProgramSummary;
|
||||
|
||||
UPROPERTY(Config)
|
||||
FString RecognitionClientKind = TEXT("http-sidecar");
|
||||
|
||||
UPROPERTY()
|
||||
TObjectPtr<UObject> ActiveRecognitionVisionClientObject;
|
||||
|
||||
UPROPERTY()
|
||||
bool bHasActiveRun = false;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
#include "CoreMinimal.h"
|
||||
#include "HyperTwistCore/HyperTwistCoreTypes.h"
|
||||
#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h"
|
||||
#include "HyperTwistReplay/HyperTwistReplayTypes.h"
|
||||
#include "HyperTwistTrainingTypes.generated.h"
|
||||
|
||||
|
|
@ -1077,6 +1078,66 @@ struct FHyperTwistTrainingRunState
|
|||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistTrainingRecognitionSessionState
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSessionOpen = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bUsingProviderBackedClient = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bHasPreviewResult = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bHasCommitResult = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bHasFinalizeResult = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ActiveSessionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString OpenedAtUtc;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastUpdatedAtUtc;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastError;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SubmittedFrameCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 CommittedObservationCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 CorrectionEventCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 FinalizedSessionCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistVisionSessionConfig SessionConfig;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistVisionServiceHealth ServiceHealth;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistVisionPreviewResult LastPreviewResult;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistVisionCommitResult LastCommitResult;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistVisionFinalizeResult LastFinalizeResult;
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistTrainingRunStepResult
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
# HyperTwist Phase 4 recognition runtime handoff packet
|
||||
|
||||
Created on `2026-05-06`
|
||||
|
||||
Status:
|
||||
|
||||
- first-party HyperTwist packet
|
||||
- bounded Phase `4` recognition runtime-handoff slice
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet closes the first honest runtime handoff after the provider-backed recognition client landed.
|
||||
|
||||
The open task is:
|
||||
|
||||
- move recognition from a sidecar contract plus standalone client into owned training runtime state, replay state, and coaching refresh flow
|
||||
|
||||
It is not:
|
||||
|
||||
- a large recognition UI pass
|
||||
- a websocket/live-stream rewrite
|
||||
- a speech or simulation packet
|
||||
- a broad refactor of the training subsystem
|
||||
|
||||
## Scope
|
||||
|
||||
Bounded lane:
|
||||
|
||||
- complete recognition session lifecycle ownership in `HyperTwistTraining`
|
||||
- expose runtime open / preview / commit / finalize / close surfaces through the subsystem, runtime library, panel widget, and session actor
|
||||
- append normalized recognition replay events into the active run packet
|
||||
- refresh repository/coaching derived state after replay mutation so existing replay-to-coach logic can wake up on live recognition data
|
||||
|
||||
Out of scope:
|
||||
|
||||
- broad dashboard redesign
|
||||
- provider-specific protocol expansion beyond the current HTTP sidecar contract
|
||||
- new gameplay content or speech flows
|
||||
- exact puzzle-state engine deepening
|
||||
|
||||
## Why this was the right next packet
|
||||
|
||||
Before this slice:
|
||||
|
||||
- the repo had a provider-backed recognition client
|
||||
- replay review analytics and recognition replay summaries already existed
|
||||
- coach-signal derivation already reacted to recognition replay ambiguity and correction pressure
|
||||
|
||||
But recognition still did not flow through the owned training runtime.
|
||||
|
||||
So the missing bridge was:
|
||||
|
||||
- own the recognition session in `HyperTwistTraining`
|
||||
- append replay events from recognition runtime activity
|
||||
- let the existing replay/training/coaching stack consume those events without inventing a second parallel lane
|
||||
|
||||
## What landed
|
||||
|
||||
Primary code changes:
|
||||
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h`
|
||||
- added explicit `CloseVisionSession(...)` to the owned vision-client contract
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp`
|
||||
- `UnrealHyperTwist/Config/DefaultGame.ini`
|
||||
- completed close-session support for both mock and provider-backed client implementations
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h`
|
||||
- added `FHyperTwistTrainingRecognitionSessionState`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp`
|
||||
- added owned runtime methods for:
|
||||
- open active recognition session
|
||||
- submit active recognition frame
|
||||
- commit active recognition observation
|
||||
- finalize active recognition session
|
||||
- close active recognition session
|
||||
- added owned recognition session state, provider-health refresh, replay-event append, and replay-mutation synchronization helpers
|
||||
- recognition replay mutation now records run state and refreshes repository/coaching views in the same subsystem pass
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.cpp`
|
||||
- exposed the new recognition runtime lane through the existing world-context runtime surface
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingPanelWidget.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingPanelWidget.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSessionActor.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSessionActor.cpp`
|
||||
- added thin runtime-consumer wrappers and cached recognition session state
|
||||
|
||||
## Runtime behavior now
|
||||
|
||||
The active training runtime now owns:
|
||||
|
||||
- recognition session open/closed state
|
||||
- provider-health inspection state
|
||||
- last preview / commit / finalize results
|
||||
- cumulative frame / commit / correction / finalize counters
|
||||
|
||||
Recognition runtime activity now mutates the active replay packet through owned event types:
|
||||
|
||||
- first frame -> `RecognitionPreview`
|
||||
- later frames -> `RecognitionScan`
|
||||
- commit warnings / missing units -> `RecognitionCorrection`
|
||||
- commit results -> `RecognitionCommit`
|
||||
- finalize outputs -> replay-seed recognition events when provided, otherwise a synthesized final commit
|
||||
|
||||
After replay mutation:
|
||||
|
||||
- the active run summary is refreshed
|
||||
- run state is persisted back into the training repository
|
||||
- repository-derived replay review analytics, recognition replay summaries, and coach signals are rebuilt in the existing same-pass flow
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- recognition session lifecycle is now owned by first-party training runtime code
|
||||
- provider-backed and mock clients both satisfy the full open/submit/commit/finalize/close contract
|
||||
- recognition runtime events reach the active replay packet
|
||||
- replay mutation wakes up the existing repository/coaching refresh path
|
||||
- the panel widget and session actor can consume the runtime lane without bypassing the subsystem
|
||||
- full product build succeeds
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. build `UnrealHyperTwist.sln` / `UnrealHyperTwistEditor`
|
||||
2. confirm the subsystem, runtime library, panel widget, and session actor all compile together
|
||||
3. confirm no Phase `3` generated-mode packet was reopened
|
||||
4. confirm the provider-backed client lane remains bounded to the owned vision contract
|
||||
|
||||
That is the packet.
|
||||
Loading…
Add table
Reference in a new issue