From d0fc383801edd853e5dd3f2276a0fc536e0f8147 Mon Sep 17 00:00:00 2001 From: axiomlogicnexus Date: Thu, 21 May 2026 06:19:02 +0200 Subject: [PATCH] Implement Phase 6R-E speech transcript session boundary --- UnrealHyperTwist/Config/DefaultGame.ini | 9 + .../HyperTwistContractLibrary.cpp | 66 +++ .../HyperTwistHttpSpeechClient.cpp | 314 ++++++++++++++ .../HyperTwistMockSpeechClient.cpp | 119 ++++++ .../HyperTwistTrainingSubsystem.cpp | 394 ++++++++++++++++++ .../HyperTwistContractLibrary.h | 9 + .../HyperTwistRecognitionTypes.h | 260 ++++++++++++ .../HyperTwistSpeechClient.h | 96 +++++ .../HyperTwistTrainingSubsystem.h | 27 ++ .../HyperTwistTrainingTypes.h | 42 ++ ...ppPhase6RESpeechTranscriptContractTest.cpp | 199 +++++++++ docs/REPO_LICENSE_TRACKING.md | 4 + ...SCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md | 159 +++++++ ...RANSCRIPT_PREPARATION_PACKET_2026-05-21.md | 182 ++++++++ .../HyperTwist/ROADMAP.md | 33 +- 15 files changed, 1905 insertions(+), 8 deletions(-) create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp create mode 100644 docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md create mode 100644 docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md diff --git a/UnrealHyperTwist/Config/DefaultGame.ini b/UnrealHyperTwist/Config/DefaultGame.ini index da77545..1cc847f 100644 --- a/UnrealHyperTwist/Config/DefaultGame.ini +++ b/UnrealHyperTwist/Config/DefaultGame.ini @@ -15,3 +15,12 @@ FinalizeSessionPath=/vision/session/finalize CloseSessionPath=/vision/session/close HealthPath=/vision/health RequestTimeoutSeconds=5.0 + +[/Script/UnrealHyperTwist.HyperTwistHttpSpeechClient] +ProviderLabel=local-http-sidecar +ServiceBaseUrl=http://127.0.0.1:8766 +OpenSessionPath=/speech/session/open +TranscribeUtterancePath=/speech/utterance +CloseSessionPath=/speech/session/close +HealthPath=/speech/health +RequestTimeoutSeconds=5.0 diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp index f11709a..0a93406 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp @@ -612,6 +612,72 @@ FHyperTwistVisionServiceHealth UHyperTwistContractLibrary::MakeMockVisionService return Health; } +FHyperTwistSpeechSessionConfig UHyperTwistContractLibrary::MakeSampleSpeechSessionConfig() +{ + FHyperTwistSpeechSessionConfig Config; + Config.SessionId = TEXT("speech_01"); + Config.ListeningContractId = TEXT("listening-threshold-lifecycle"); + Config.InputRouteId = TEXT("queue/listening"); + Config.AudioEncoding = TEXT("pcm-f32le-16khz-mono"); + Config.SampleRateHz = 16000; + Config.ChannelCount = 1; + Config.LanguageMode = TEXT("auto"); + Config.TaskKind = TEXT("transcribe"); + Config.GrammarProfileId = TEXT("coach-command-grammar-v1"); + Config.bEnableVad = true; + Config.bEnableWordTimestamps = false; + Config.VadPolicy.Threshold = 0.5f; + Config.VadPolicy.MinSpeechDurationMs = 250; + Config.VadPolicy.MinSilenceDurationMs = 200; + Config.VadPolicy.MaxSpeechDurationMs = 12000; + Config.VadPolicy.SpeechPadMs = 100; + Config.CommandHints = { + TEXT("ready coach"), + TEXT("repeat case"), + TEXT("next case") + }; + return Config; +} + +FHyperTwistSpeechTranscriptResult UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult() +{ + FHyperTwistSpeechTranscriptResult Result; + Result.SessionId = TEXT("speech_01"); + Result.UtteranceId = TEXT("speech_01_utterance_001"); + Result.LanguageCode = TEXT("en"); + Result.TaskKind = TEXT("transcribe"); + Result.GrammarProfileId = TEXT("coach-command-grammar-v1"); + Result.bIsFinal = true; + Result.TranscriptText = TEXT("ready coach"); + + FHyperTwistSpeechTranscriptSegment Segment; + Segment.SegmentOrdinal = 0; + Segment.Text = Result.TranscriptText; + Segment.StartMs = 120; + Segment.EndMs = 840; + Segment.Confidence = 0.94f; + Segment.NoSpeechProbability = 0.04f; + Segment.bGrammarConstrained = true; + Result.Segments = {Segment}; + return Result; +} + +FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechServiceHealth() +{ + FHyperTwistSpeechServiceHealth Health; + Health.ProviderLabel = TEXT("mock"); + Health.ServiceVersion = TEXT("mock-speech/v1"); + Health.ServiceEndpoint = TEXT("in-process://mock"); + Health.Capabilities = { + TEXT("transcribe"), + TEXT("vadSegments"), + TEXT("grammarHints"), + TEXT("sessioned-mock-client") + }; + Health.bReady = true; + return Health; +} + FHyperTwistContentPack UHyperTwistContractLibrary::MakeSampleAlgTrainerContentPack() { return UHyperTwistTrainingCatalogLibrary::MakeAlgTrainerStarterContentPack(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp new file mode 100644 index 0000000..d9d31cf --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp @@ -0,0 +1,314 @@ +#include "HyperTwistRecognition/HyperTwistSpeechClient.h" + +#include "Dom/JsonObject.h" +#include "Http.h" +#include "HttpManager.h" +#include "HttpModule.h" +#include "Interfaces/IHttpRequest.h" +#include "Interfaces/IHttpResponse.h" +#include "JsonObjectConverter.h" +#include "Serialization/JsonSerializer.h" +#include "Serialization/JsonWriter.h" + +namespace HyperTwistHttpSpeechClientInternal +{ + template + bool SerializeStruct(const TStruct& Value, FString& OutJson) + { + return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0); + } + + template + bool DeserializeStruct(const FString& Json, TStruct& OutValue) + { + return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0); + } + + FString MakeSessionIdJson(const FString& SessionId) + { + FString Json; + const TSharedRef Root = MakeShared(); + Root->SetStringField(TEXT("sessionId"), SessionId); + const TSharedRef> Writer = TJsonWriterFactory<>::Create(&Json); + FJsonSerializer::Serialize(Root, Writer); + return Json; + } + + FString BuildUrl(const FString& BaseUrl, const FString& Path) + { + if (BaseUrl.IsEmpty()) + { + return FString(); + } + + const FString TrimmedBase = BaseUrl.EndsWith(TEXT("/")) ? BaseUrl.LeftChop(1) : BaseUrl; + if (Path.IsEmpty()) + { + return TrimmedBase; + } + + const FString TrimmedPath = Path.StartsWith(TEXT("/")) ? Path.RightChop(1) : Path; + return FString::Printf(TEXT("%s/%s"), *TrimmedBase, *TrimmedPath); + } + + struct FHttpJsonResponse + { + bool bCompleted = false; + bool bSucceeded = false; + int32 StatusCode = 0; + FString ResponseBody; + FString Error; + }; + + bool ExecuteJsonRequest( + const UHyperTwistHttpSpeechClient& Client, + const FString& Verb, + const FString& Url, + const FString& RequestJson, + FHttpJsonResponse& OutResponse) + { + OutResponse = FHttpJsonResponse(); + + if (Url.IsEmpty()) + { + OutResponse.Error = TEXT("service-base-url-missing"); + return false; + } + + const TSharedRef Request = FHttpModule::Get().CreateRequest(); + Request->SetURL(Url); + Request->SetVerb(Verb); + Request->SetHeader(TEXT("Accept"), TEXT("application/json")); + Request->SetHeader(TEXT("Content-Type"), TEXT("application/json")); + + if (!Client.AuthorizationToken.IsEmpty()) + { + Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *Client.AuthorizationToken)); + } + + if (!RequestJson.IsEmpty()) + { + Request->SetContentAsString(RequestJson); + } + + Request->OnProcessRequestComplete().BindLambda( + [&OutResponse](FHttpRequestPtr, FHttpResponsePtr Response, bool bWasSuccessful) + { + OutResponse.bCompleted = true; + + if (Response.IsValid()) + { + OutResponse.StatusCode = Response->GetResponseCode(); + OutResponse.ResponseBody = Response->GetContentAsString(); + } + + if (!bWasSuccessful) + { + OutResponse.Error = TEXT("request-failed"); + return; + } + + if (!Response.IsValid()) + { + OutResponse.Error = TEXT("response-missing"); + return; + } + + if (!EHttpResponseCodes::IsOk(OutResponse.StatusCode)) + { + OutResponse.Error = FString::Printf(TEXT("http-%d"), OutResponse.StatusCode); + return; + } + + OutResponse.bSucceeded = true; + }); + + if (!Request->ProcessRequest()) + { + OutResponse.Error = TEXT("request-start-failed"); + return false; + } + + const double TimeoutSeconds = FMath::Max(static_cast(Client.RequestTimeoutSeconds), 0.1); + const double Deadline = FPlatformTime::Seconds() + TimeoutSeconds; + while (!OutResponse.bCompleted && FPlatformTime::Seconds() < Deadline) + { + FHttpModule::Get().GetHttpManager().Tick(0.01f); + FPlatformProcess::Sleep(0.01f); + } + + if (!OutResponse.bCompleted) + { + Request->CancelRequest(); + OutResponse.Error = TEXT("request-timeout"); + return false; + } + + return OutResponse.bSucceeded; + } +} + +bool UHyperTwistHttpSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) +{ + OutError.Reset(); + + FString RequestJson; + if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Config, RequestJson)) + { + OutError = TEXT("request-serialize-failed"); + LastTransportError = OutError; + return false; + } + + HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response; + if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest( + *this, + TEXT("POST"), + HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, OpenSessionPath), + RequestJson, + Response)) + { + OutError = Response.Error; + LastTransportError = OutError; + return false; + } + + LastTransportError.Reset(); + return true; +} + +FHyperTwistSpeechTranscriptResult UHyperTwistHttpSpeechClient::TranscribeSpeechUtterance( + const FHyperTwistSpeechUtteranceEnvelope& Utterance +) +{ + FHyperTwistSpeechTranscriptResult Result; + Result.SessionId = Utterance.SessionId; + Result.UtteranceId = Utterance.UtteranceId; + + FString RequestJson; + if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Utterance, RequestJson)) + { + LastTransportError = TEXT("request-serialize-failed"); + Result.Warnings.Add(LastTransportError); + return Result; + } + + HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response; + if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest( + *this, + TEXT("POST"), + HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, TranscribeUtterancePath), + RequestJson, + Response)) + { + LastTransportError = Response.Error; + Result.Warnings.Add(LastTransportError); + return Result; + } + + if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Result)) + { + LastTransportError = TEXT("invalid-provider-response"); + Result = FHyperTwistSpeechTranscriptResult(); + Result.SessionId = Utterance.SessionId; + Result.UtteranceId = Utterance.UtteranceId; + Result.Warnings.Add(LastTransportError); + return Result; + } + + if (Result.SessionId.IsEmpty()) + { + Result.SessionId = Utterance.SessionId; + } + if (Result.UtteranceId.IsEmpty()) + { + Result.UtteranceId = Utterance.UtteranceId; + } + + LastTransportError.Reset(); + return Result; +} + +bool UHyperTwistHttpSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError) +{ + OutError.Reset(); + + HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response; + if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest( + *this, + TEXT("POST"), + HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath), + HyperTwistHttpSpeechClientInternal::MakeSessionIdJson(SessionId), + Response)) + { + OutError = Response.Error; + LastTransportError = OutError; + return false; + } + + LastTransportError.Reset(); + return true; +} + +FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHealth() const +{ + FHyperTwistSpeechServiceHealth Health; + Health.ProviderLabel = ProviderLabel; + Health.ServiceVersion = TEXT("provider-unavailable/v1"); + Health.ServiceEndpoint = ServiceBaseUrl; + Health.bReady = false; + Health.Capabilities = { + TEXT("transport:http"), + TEXT("provider-backed-speech"), + TEXT("transcribe"), + TEXT("vadSegments"), + TEXT("grammarHints") + }; + + HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response; + if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest( + *this, + TEXT("GET"), + HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, HealthPath), + FString(), + Response)) + { + LastTransportError = Response.Error; + Health.LastError = LastTransportError; + return Health; + } + + if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Health)) + { + LastTransportError = TEXT("invalid-provider-response"); + Health = FHyperTwistSpeechServiceHealth(); + Health.ProviderLabel = ProviderLabel; + Health.ServiceVersion = TEXT("provider-unavailable/v1"); + Health.ServiceEndpoint = ServiceBaseUrl; + Health.bReady = false; + Health.Capabilities = { + TEXT("transport:http"), + TEXT("provider-backed-speech"), + TEXT("transcribe"), + TEXT("vadSegments"), + TEXT("grammarHints") + }; + Health.LastError = LastTransportError; + return Health; + } + + if (Health.ProviderLabel.IsEmpty()) + { + Health.ProviderLabel = ProviderLabel; + } + if (Health.ServiceEndpoint.IsEmpty()) + { + Health.ServiceEndpoint = ServiceBaseUrl; + } + + Health.Capabilities.AddUnique(TEXT("transport:http")); + Health.Capabilities.AddUnique(TEXT("provider-backed-speech")); + LastTransportError.Reset(); + Health.LastError.Reset(); + return Health; +} diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp new file mode 100644 index 0000000..56a99f6 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp @@ -0,0 +1,119 @@ +#include "HyperTwistRecognition/HyperTwistSpeechClient.h" + +#include "HyperTwistBootstrap/HyperTwistContractLibrary.h" + +namespace HyperTwistMockSpeechClientInternal +{ + FString ResolveTranscriptText( + const FHyperTwistSpeechSessionConfig* SessionConfig, + const FHyperTwistSpeechUtteranceEnvelope& Utterance + ) + { + const FString NormalizedAudioRef = Utterance.AudioRef.ToLower(); + if (NormalizedAudioRef.Contains(TEXT("repeat-case"))) + { + return TEXT("repeat case"); + } + if (NormalizedAudioRef.Contains(TEXT("next-case"))) + { + return TEXT("next case"); + } + if (NormalizedAudioRef.Contains(TEXT("ready-coach"))) + { + return TEXT("ready coach"); + } + + if (Utterance.LocalCommandHints.Num() > 0 && !Utterance.LocalCommandHints[0].IsEmpty()) + { + return Utterance.LocalCommandHints[0]; + } + + if (SessionConfig != nullptr && SessionConfig->CommandHints.Num() > 0 && !SessionConfig->CommandHints[0].IsEmpty()) + { + return SessionConfig->CommandHints[0]; + } + + return TEXT("ready coach"); + } +} + +bool UHyperTwistMockSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) +{ + OutError.Reset(); + + if (!Config.IsStructurallyValid()) + { + OutError = TEXT("speech-session-config-invalid"); + return false; + } + + SessionConfigs.Add(Config.SessionId, Config); + SessionUtteranceCounts.Add(Config.SessionId, 0); + return true; +} + +FHyperTwistSpeechTranscriptResult UHyperTwistMockSpeechClient::TranscribeSpeechUtterance( + const FHyperTwistSpeechUtteranceEnvelope& Utterance +) +{ + FHyperTwistSpeechTranscriptResult Result = UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult(); + Result.SessionId = Utterance.SessionId; + Result.UtteranceId = Utterance.UtteranceId; + + const FHyperTwistSpeechSessionConfig* SessionConfig = SessionConfigs.Find(Utterance.SessionId); + if (SessionConfig == nullptr) + { + Result.Warnings.Add(TEXT("session-not-open")); + return Result; + } + + const FString TranscriptText = + HyperTwistMockSpeechClientInternal::ResolveTranscriptText(SessionConfig, Utterance); + + SessionUtteranceCounts.Add(Utterance.SessionId, SessionUtteranceCounts.FindRef(Utterance.SessionId) + 1); + + Result.LanguageCode = SessionConfig->LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase) + ? TEXT("en") + : SessionConfig->LanguageMode; + Result.TaskKind = SessionConfig->TaskKind; + Result.GrammarProfileId = SessionConfig->GrammarProfileId; + Result.bIsFinal = true; + Result.TranscriptText = TranscriptText; + Result.Segments.Reset(); + + FHyperTwistSpeechTranscriptSegment Segment; + Segment.SegmentOrdinal = 0; + Segment.Text = TranscriptText; + Segment.StartMs = FMath::Max(0, Utterance.SpeechStartMs); + Segment.EndMs = FMath::Max(Segment.StartMs, Utterance.SpeechEndMs); + Segment.Confidence = 0.93f; + Segment.NoSpeechProbability = 0.05f; + Segment.bGrammarConstrained = !SessionConfig->GrammarProfileId.IsEmpty(); + Result.Segments.Add(Segment); + + return Result; +} + +bool UHyperTwistMockSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError) +{ + OutError.Reset(); + + if (!SessionConfigs.Contains(SessionId)) + { + OutError = TEXT("session-not-open"); + return false; + } + + SessionConfigs.Remove(SessionId); + SessionUtteranceCounts.Remove(SessionId); + return true; +} + +FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHealth() const +{ + FHyperTwistSpeechServiceHealth Health = UHyperTwistContractLibrary::MakeMockSpeechServiceHealth(); + Health.ProviderLabel = TEXT("mock"); + Health.ServiceEndpoint = TEXT("in-process://mock"); + Health.Capabilities.AddUnique(TEXT("offline-transcribe")); + return Health; +} diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp index ec4e52a..707c313 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp @@ -1,6 +1,7 @@ #include "HyperTwistTraining/HyperTwistTrainingSubsystem.h" #include "HyperTwistRecognition/HyperTwistVisionClient.h" +#include "HyperTwistRecognition/HyperTwistSpeechClient.h" #include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h" #include "HyperTwistReplay/HyperTwistReplayLibrary.h" #include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h" @@ -11,6 +12,7 @@ #include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingPublicationLibrary.h" #include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h" +#include "HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h" #include "JsonObjectConverter.h" namespace HyperTwistTrainingSubsystemInternal @@ -1855,17 +1857,35 @@ namespace HyperTwistTrainingSubsystemInternal : nullptr; } + IHyperTwistSpeechClient* ResolveSpeechClient(UObject* Candidate) + { + return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistSpeechClient::StaticClass()) + ? Cast(Candidate) + : nullptr; + } + FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal) { const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal); return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal); } + FString BuildSpeechSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal) + { + const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal); + return FString::Printf(TEXT("%s_speech_%02d"), *Session.TrainingSessionId, ResolvedOrdinal); + } + FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal) { return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal)); } + FString BuildSpeechUtteranceId(const FString& SessionId, const int32 UtteranceOrdinal) + { + return FString::Printf(TEXT("%s_utterance_%03d"), *SessionId, FMath::Max(1, UtteranceOrdinal)); + } + bool IsQbrClassicCubeRoute(const FHyperTwistVisionSessionConfig& SessionConfig) { return SessionConfig.ExpectedPuzzleFamily == EHyperTwistPuzzleFamily::ClassicCube @@ -3609,6 +3629,11 @@ FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActi return ActiveRecognitionSessionState; } +FHyperTwistTrainingCompanionSpeechSessionState UHyperTwistTrainingSubsystem::GetActiveCompanionSpeechSessionState() const +{ + return ActiveCompanionSpeechSessionState; +} + TArray UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const { return ActiveCoachSignals; @@ -5487,7 +5512,13 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe FString CloseError; CloseActiveRecognitionSession(CloseError); } + if (ActiveCompanionSpeechSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveCompanionSpeechSession(CloseError); + } ResetRecognitionSessionState(); + ResetCompanionSpeechSessionState(); ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState(); ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus(); ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan(); @@ -6360,6 +6391,16 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool bAbortSession ? TEXT("training-run-aborted") : TEXT("training-run-completed") ); } + if (ActiveRecognitionSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveRecognitionSession(CloseError); + } + if (ActiveCompanionSpeechSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveCompanionSpeechSession(CloseError); + } ResetActiveLiveTimerState(); @@ -6418,6 +6459,7 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool } } ResetRecognitionSessionState(); + ResetCompanionSpeechSessionState(); RefreshRepositoryViews(); return ActiveRunState.Session; } @@ -6429,6 +6471,11 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun() FString CloseError; CloseActiveRecognitionSession(CloseError); } + if (ActiveCompanionSpeechSessionState.bSessionOpen) + { + FString CloseError; + CloseActiveCompanionSpeechSession(CloseError); + } if (HasActiveSmartDeviceSession()) { StopActiveSmartDeviceSession(TEXT("training-run-cleared")); @@ -6466,6 +6513,7 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun() ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan(); bHasActiveRun = false; ResetRecognitionSessionState(); + ResetCompanionSpeechSessionState(); RefreshRepositoryViews(); } @@ -7043,6 +7091,226 @@ bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutErr return true; } +bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& OutError) +{ + OutError.Reset(); + + if (!HasActiveRun()) + { + OutError = TEXT("no-active-training-run"); + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient( + ResolveCompanionSpeechClientObject() + ); + if (SpeechClient == nullptr) + { + OutError = TEXT("speech-client-unavailable"); + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + const FHyperTwistSpeechSessionConfig SessionConfig = BuildActiveCompanionSpeechSessionConfig(); + if (SessionConfig.SessionId.IsEmpty()) + { + OutError = TEXT("speech-session-id-unavailable"); + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + if (ActiveCompanionSpeechSessionState.bSessionOpen + && ActiveCompanionSpeechSessionState.ActiveSessionId == SessionConfig.SessionId) + { + ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig; + ActiveCompanionSpeechSessionState.LastError.Reset(); + ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset(); + return true; + } + + if (ActiveCompanionSpeechSessionState.bSessionOpen + && ActiveCompanionSpeechSessionState.ActiveSessionId != SessionConfig.SessionId) + { + FString CloseError; + SpeechClient->CloseSpeechSession(ActiveCompanionSpeechSessionState.ActiveSessionId, CloseError); + } + + if (!SpeechClient->OpenSpeechSession(SessionConfig, OutError)) + { + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveCompanionSpeechSessionState.bSessionOpen = true; + ActiveCompanionSpeechSessionState.bUsingProviderBackedClient = + ActiveCompanionSpeechClientObject != nullptr + && ActiveCompanionSpeechClientObject->IsA(); + ActiveCompanionSpeechSessionState.ActiveSessionId = SessionConfig.SessionId; + ActiveCompanionSpeechSessionState.OpenedAtUtc = OpenedAtUtc; + ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = OpenedAtUtc; + ActiveCompanionSpeechSessionState.LastError.Reset(); + ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig; + ActiveCompanionSpeechSessionState.bHasTranscriptResult = false; + ActiveCompanionSpeechSessionState.LastTranscriptResult = FHyperTwistSpeechTranscriptResult(); + RefreshCompanionSpeechServiceHealth(); + return true; +} + +FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveCompanionSpeechUtterance( + const FHyperTwistSpeechUtteranceEnvelope& Utterance +) +{ + FHyperTwistSpeechTranscriptResult Result; + + if (!HasActiveRun()) + { + Result.Warnings.Add(TEXT("no-active-training-run")); + return Result; + } + + FString OpenError; + if (!OpenActiveCompanionSpeechSession(OpenError)) + { + Result.SessionId = !Utterance.SessionId.IsEmpty() + ? Utterance.SessionId + : BuildActiveCompanionSpeechSessionConfig().SessionId; + Result.UtteranceId = Utterance.UtteranceId; + Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("speech-session-open-failed") : OpenError); + return Result; + } + + IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient( + ResolveCompanionSpeechClientObject() + ); + if (SpeechClient == nullptr) + { + Result.Warnings.Add(TEXT("speech-client-unavailable")); + ActiveCompanionSpeechSessionState.LastError = Result.Warnings[0]; + ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError; + return Result; + } + + FHyperTwistSpeechUtteranceEnvelope NormalizedUtterance = Utterance; + NormalizedUtterance.SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId; + if (NormalizedUtterance.UtteranceId.IsEmpty()) + { + NormalizedUtterance.UtteranceId = HyperTwistTrainingSubsystemInternal::BuildSpeechUtteranceId( + NormalizedUtterance.SessionId, + ActiveCompanionSpeechSessionState.SubmittedUtteranceCount + 1 + ); + } + if (NormalizedUtterance.CapturedAtUtc.IsEmpty()) + { + NormalizedUtterance.CapturedAtUtc = FDateTime::UtcNow().ToIso8601(); + } + if (NormalizedUtterance.SampleRateHz <= 0) + { + NormalizedUtterance.SampleRateHz = ActiveCompanionSpeechSessionState.SessionConfig.SampleRateHz; + } + if (NormalizedUtterance.ChannelCount <= 0) + { + NormalizedUtterance.ChannelCount = ActiveCompanionSpeechSessionState.SessionConfig.ChannelCount; + } + if (NormalizedUtterance.LocalCommandHints.Num() <= 0) + { + NormalizedUtterance.LocalCommandHints = ActiveCompanionSpeechSessionState.SessionConfig.CommandHints; + } + NormalizedUtterance.SilenceGapMs = FMath::Max(0, NormalizedUtterance.SilenceGapMs); + NormalizedUtterance.SpeechStartMs = FMath::Max(0, NormalizedUtterance.SpeechStartMs); + NormalizedUtterance.SpeechEndMs = FMath::Max( + NormalizedUtterance.SpeechStartMs, + NormalizedUtterance.SpeechEndMs + ); + + Result = SpeechClient->TranscribeSpeechUtterance(NormalizedUtterance); + if (Result.SessionId.IsEmpty()) + { + Result.SessionId = NormalizedUtterance.SessionId; + } + if (Result.UtteranceId.IsEmpty()) + { + Result.UtteranceId = NormalizedUtterance.UtteranceId; + } + if (Result.TaskKind.IsEmpty()) + { + Result.TaskKind = ActiveCompanionSpeechSessionState.SessionConfig.TaskKind; + } + if (Result.GrammarProfileId.IsEmpty()) + { + Result.GrammarProfileId = ActiveCompanionSpeechSessionState.SessionConfig.GrammarProfileId; + } + if (Result.TranscriptText.IsEmpty() && Result.Segments.Num() > 0) + { + TArray SegmentTexts; + for (const FHyperTwistSpeechTranscriptSegment& Segment : Result.Segments) + { + if (!Segment.Text.IsEmpty()) + { + SegmentTexts.Add(Segment.Text); + } + } + Result.TranscriptText = FString::Join(SegmentTexts, TEXT(" ")); + } + + ++ActiveCompanionSpeechSessionState.SubmittedUtteranceCount; + if (Result.bIsFinal && Result.IsStructurallyValid()) + { + ++ActiveCompanionSpeechSessionState.FinalTranscriptCount; + } + ActiveCompanionSpeechSessionState.bHasTranscriptResult = true; + ActiveCompanionSpeechSessionState.LastTranscriptResult = Result; + ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveCompanionSpeechSessionState.LastError = + Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString(); + ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError; + + return Result; +} + +bool UHyperTwistTrainingSubsystem::CloseActiveCompanionSpeechSession(FString& OutError) +{ + OutError.Reset(); + + if (!ActiveCompanionSpeechSessionState.bSessionOpen || ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty()) + { + RefreshCompanionSpeechServiceHealth(); + return true; + } + + IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient( + ResolveCompanionSpeechClientObject() + ); + if (SpeechClient == nullptr) + { + OutError = TEXT("speech-client-unavailable"); + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + const FString SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId; + const bool bClosed = SpeechClient->CloseSpeechSession(SessionId, OutError); + if (!bClosed && OutError != TEXT("session-not-open")) + { + ActiveCompanionSpeechSessionState.LastError = OutError; + RefreshCompanionSpeechServiceHealth(); + return false; + } + + ActiveCompanionSpeechSessionState.bSessionOpen = false; + ActiveCompanionSpeechSessionState.ActiveSessionId.Reset(); + ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); + ActiveCompanionSpeechSessionState.LastError.Reset(); + RefreshCompanionSpeechServiceHealth(); + return true; +} + void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun() { ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState(); @@ -7410,6 +7678,74 @@ FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognit return SessionConfig; } +FHyperTwistSpeechSessionConfig UHyperTwistTrainingSubsystem::BuildActiveCompanionSpeechSessionConfig() const +{ + FHyperTwistSpeechSessionConfig SessionConfig = ActiveCompanionSpeechSessionState.SessionConfig; + if (!HasActiveRun()) + { + return SessionConfig; + } + + const FHyperTwistTrainingCompanionReferenceBundle CompanionBundle = + UHyperTwistTrainingRuntimeLibrary::GetBundledEmbodiedCompanionReferenceBundle(); + + SessionConfig.SessionId = !ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty() + ? ActiveCompanionSpeechSessionState.ActiveSessionId + : HyperTwistTrainingSubsystemInternal::BuildSpeechSessionId(ActiveRunState.Session, 1); + SessionConfig.ListeningContractId = !CompanionBundle.PrimaryListeningContractId.IsEmpty() + ? CompanionBundle.PrimaryListeningContractId + : TEXT("listening-threshold-lifecycle"); + if (SessionConfig.InputRouteId.IsEmpty()) + { + SessionConfig.InputRouteId = TEXT("queue/listening"); + } + if (SessionConfig.AudioEncoding.IsEmpty()) + { + SessionConfig.AudioEncoding = TEXT("pcm-f32le-16khz-mono"); + } + if (SessionConfig.SampleRateHz <= 0) + { + SessionConfig.SampleRateHz = 16000; + } + if (SessionConfig.ChannelCount <= 0) + { + SessionConfig.ChannelCount = 1; + } + if (SessionConfig.LanguageMode.IsEmpty()) + { + SessionConfig.LanguageMode = TEXT("auto"); + } + if (SessionConfig.TaskKind.IsEmpty()) + { + SessionConfig.TaskKind = TEXT("transcribe"); + } + if (SessionConfig.GrammarProfileId.IsEmpty()) + { + SessionConfig.GrammarProfileId = TEXT("coach-command-grammar-v1"); + } + SessionConfig.bEnableVad = true; + SessionConfig.bEnableWordTimestamps = false; + if (!SessionConfig.VadPolicy.IsStructurallyValid()) + { + SessionConfig.VadPolicy.Threshold = 0.5f; + SessionConfig.VadPolicy.MinSpeechDurationMs = 250; + SessionConfig.VadPolicy.MinSilenceDurationMs = 200; + SessionConfig.VadPolicy.MaxSpeechDurationMs = 12000; + SessionConfig.VadPolicy.SpeechPadMs = 100; + } + if (SessionConfig.CommandHints.Num() <= 0) + { + SessionConfig.CommandHints = { + TEXT("ready coach"), + TEXT("repeat case"), + TEXT("next case"), + TEXT("pause coaching") + }; + } + + return SessionConfig; +} + void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth() { IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient( @@ -7435,6 +7771,58 @@ void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth() ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth(); } +UObject* UHyperTwistTrainingSubsystem::ResolveCompanionSpeechClientObject() +{ + LoadConfig(); + + const bool bUseMockClient = CompanionSpeechClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase); + const bool bNeedsNewClient = ActiveCompanionSpeechClientObject == nullptr + || (bUseMockClient && !ActiveCompanionSpeechClientObject->IsA()) + || (!bUseMockClient && !ActiveCompanionSpeechClientObject->IsA()); + if (bNeedsNewClient) + { + if (bUseMockClient) + { + ActiveCompanionSpeechClientObject = NewObject(this); + } + else + { + UHyperTwistHttpSpeechClient* HttpSpeechClient = NewObject(this); + HttpSpeechClient->LoadConfig(); + ActiveCompanionSpeechClientObject = HttpSpeechClient; + } + } + + return ActiveCompanionSpeechClientObject; +} + +void UHyperTwistTrainingSubsystem::RefreshCompanionSpeechServiceHealth() +{ + IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient( + ResolveCompanionSpeechClientObject() + ); + ActiveCompanionSpeechSessionState.bUsingProviderBackedClient = + ActiveCompanionSpeechClientObject != nullptr + && ActiveCompanionSpeechClientObject->IsA(); + if (SpeechClient == nullptr) + { + ActiveCompanionSpeechSessionState.ServiceHealth = FHyperTwistSpeechServiceHealth(); + ActiveCompanionSpeechSessionState.ServiceHealth.ProviderLabel = + ActiveCompanionSpeechSessionState.bUsingProviderBackedClient + ? TEXT("local-http-sidecar") + : TEXT("mock"); + ActiveCompanionSpeechSessionState.ServiceHealth.ServiceEndpoint = + ActiveCompanionSpeechSessionState.bUsingProviderBackedClient + ? TEXT("http://127.0.0.1:8766") + : TEXT("in-process://mock"); + ActiveCompanionSpeechSessionState.ServiceHealth.bReady = false; + ActiveCompanionSpeechSessionState.ServiceHealth.LastError = TEXT("speech-client-unavailable"); + return; + } + + ActiveCompanionSpeechSessionState.ServiceHealth = SpeechClient->GetSpeechServiceHealth(); +} + void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent( const EHyperTwistReplayEventType EventType, const FHyperTwistReplayRecognitionPayload& Payload, @@ -7492,6 +7880,12 @@ void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState() RefreshRecognitionServiceHealth(); } +void UHyperTwistTrainingSubsystem::ResetCompanionSpeechSessionState() +{ + ActiveCompanionSpeechSessionState = FHyperTwistTrainingCompanionSpeechSessionState(); + RefreshCompanionSpeechServiceHealth(); +} + bool UHyperTwistTrainingSubsystem::TryResolveRepositoryViewContext( FString& OutUserId, FString& OutDeckId, diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h index 3f3c996..9e2f176 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h @@ -57,6 +57,15 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") static FHyperTwistVisionServiceHealth MakeMockVisionServiceHealth(); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistSpeechSessionConfig MakeSampleSpeechSessionConfig(); + + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistSpeechTranscriptResult MakeMockSpeechTranscriptResult(); + + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistSpeechServiceHealth MakeMockSpeechServiceHealth(); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training") static FHyperTwistContentPack MakeSampleAlgTrainerContentPack(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h index dc90878..1a7d3c7 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h @@ -572,3 +572,263 @@ struct FHyperTwistVisionServiceHealth UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") bool bReady = true; }; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechVadPolicy +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float Threshold = 0.5f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 MinSpeechDurationMs = 250; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 MinSilenceDurationMs = 200; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 MaxSpeechDurationMs = 12000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SpeechPadMs = 100; + + bool IsStructurallyValid() const + { + return Threshold > 0.0f + && MinSpeechDurationMs >= 0 + && MinSilenceDurationMs >= 0 + && MaxSpeechDurationMs >= MinSpeechDurationMs + && SpeechPadMs >= 0; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechSessionConfig +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SessionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ListeningContractId = TEXT("listening-threshold-lifecycle"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString InputRouteId = TEXT("queue/listening"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString AudioEncoding = TEXT("pcm-f32le-16khz-mono"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SampleRateHz = 16000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 ChannelCount = 1; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageMode = TEXT("auto"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString TaskKind = TEXT("transcribe"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString GrammarProfileId = TEXT("coach-command-grammar-v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bEnableVad = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bEnableWordTimestamps = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechVadPolicy VadPolicy; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray CommandHints; + + bool IsStructurallyValid() const + { + return !SessionId.IsEmpty() + && !ListeningContractId.IsEmpty() + && !InputRouteId.IsEmpty() + && !AudioEncoding.IsEmpty() + && SampleRateHz > 0 + && ChannelCount > 0 + && !TaskKind.IsEmpty() + && VadPolicy.IsStructurallyValid(); + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechUtteranceEnvelope +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SessionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString UtteranceId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CapturedAtUtc; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString AudioRef; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SourceTimestampMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SampleCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SampleRateHz = 16000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 ChannelCount = 1; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SpeechStartMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SpeechEndMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SilenceGapMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float EnergyThreshold = 0.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray LocalCommandHints; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSpeechCommitReady = true; + + bool IsStructurallyValid() const + { + return !SessionId.IsEmpty() + && !UtteranceId.IsEmpty() + && SampleRateHz > 0 + && ChannelCount > 0 + && SpeechStartMs >= 0 + && SpeechEndMs >= SpeechStartMs + && SilenceGapMs >= 0; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechTranscriptSegment +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SegmentOrdinal = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString Text; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 StartMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 EndMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float Confidence = 0.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float NoSpeechProbability = 0.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bGrammarConstrained = false; + + bool IsStructurallyValid() const + { + return SegmentOrdinal >= 0 + && !Text.IsEmpty() + && StartMs >= 0 + && EndMs >= StartMs + && Confidence >= 0.0f; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechTranscriptResult +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SessionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString UtteranceId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageCode = TEXT("en"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString TaskKind = TEXT("transcribe"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString GrammarProfileId = TEXT("coach-command-grammar-v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bIsFinal = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString TranscriptText; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Segments; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Warnings; + + bool IsStructurallyValid() const + { + if (SessionId.IsEmpty() + || UtteranceId.IsEmpty() + || TaskKind.IsEmpty() + || (TranscriptText.IsEmpty() && Segments.Num() <= 0)) + { + return false; + } + + for (const FHyperTwistSpeechTranscriptSegment& Segment : Segments) + { + if (!Segment.IsStructurallyValid()) + { + return false; + } + } + + return true; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechServiceHealth +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ProviderLabel = TEXT("mock"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceVersion = TEXT("mock-speech/v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceEndpoint; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Capabilities; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastError; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bReady = true; +}; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h new file mode 100644 index 0000000..540dd4a --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h @@ -0,0 +1,96 @@ +#pragma once + +#include "CoreMinimal.h" +#include "UObject/Interface.h" +#include "UObject/Object.h" +#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h" +#include "HyperTwistSpeechClient.generated.h" + +UINTERFACE(BlueprintType) +class UNREALHYPERTWIST_API UHyperTwistSpeechClient : public UInterface +{ + GENERATED_BODY() +}; + +class UNREALHYPERTWIST_API IHyperTwistSpeechClient +{ + GENERATED_BODY() + +public: + virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) = 0; + virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) = 0; + virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) = 0; + virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const = 0; +}; + +UCLASS(BlueprintType) +class UNREALHYPERTWIST_API UHyperTwistMockSpeechClient : public UObject, public IHyperTwistSpeechClient +{ + GENERATED_BODY() + +public: + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override; + +private: + UPROPERTY() + TMap SessionConfigs; + + UPROPERTY() + TMap SessionUtteranceCounts; +}; + +UCLASS(BlueprintType, Config = Game, DefaultConfig) +class UNREALHYPERTWIST_API UHyperTwistHttpSpeechClient : public UObject, public IHyperTwistSpeechClient +{ + GENERATED_BODY() + +public: + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString ProviderLabel = TEXT("local-http-sidecar"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString ServiceBaseUrl = TEXT("http://127.0.0.1:8766"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString OpenSessionPath = TEXT("/speech/session/open"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString TranscribeUtterancePath = TEXT("/speech/utterance"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString CloseSessionPath = TEXT("/speech/session/close"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString HealthPath = TEXT("/speech/health"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + float RequestTimeoutSeconds = 5.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString AuthorizationToken; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override; + +private: + mutable FString LastTransportError; +}; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h index 80d28b6..5cfad6c 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h @@ -176,6 +176,9 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition") FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const; + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Speech") + FHyperTwistTrainingCompanionSpeechSessionState GetActiveCompanionSpeechSessionState() const; + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach") TArray GetActiveCoachSignals() const; @@ -685,6 +688,17 @@ public: UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition") bool CloseActiveRecognitionSession(FString& OutError); + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech") + bool OpenActiveCompanionSpeechSession(FString& OutError); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech") + FHyperTwistSpeechTranscriptResult SubmitActiveCompanionSpeechUtterance( + const FHyperTwistSpeechUtteranceEnvelope& Utterance + ); + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech") + bool CloseActiveCompanionSpeechSession(FString& OutError); + void SetActiveAlgJsSequence(const FHyperTwistAlgorithmSequence& Sequence); bool TryGetActiveAlgJsSequence(FHyperTwistAlgorithmSequence& OutSequence) const; @@ -716,8 +730,11 @@ private: const FString& NowUtc ) const; FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const; + FHyperTwistSpeechSessionConfig BuildActiveCompanionSpeechSessionConfig() const; UObject* ResolveRecognitionVisionClientObject(); + UObject* ResolveCompanionSpeechClientObject(); void RefreshRecognitionServiceHealth(); + void RefreshCompanionSpeechServiceHealth(); void AppendRecognitionReplayEvent( EHyperTwistReplayEventType EventType, const FHyperTwistReplayRecognitionPayload& Payload, @@ -725,6 +742,7 @@ private: ); void SynchronizeRecognitionReplayMutation(); void ResetRecognitionSessionState(); + void ResetCompanionSpeechSessionState(); TArray BuildTrainingSessionTemplatesForUser( const FString& UserId, const FString& ReferenceUtc @@ -820,6 +838,9 @@ private: UPROPERTY() FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState; + UPROPERTY() + FHyperTwistTrainingCompanionSpeechSessionState ActiveCompanionSpeechSessionState; + UPROPERTY() TArray ActiveCoachSignals; @@ -898,6 +919,12 @@ private: UPROPERTY() TObjectPtr ActiveRecognitionVisionClientObject; + UPROPERTY(Config) + FString CompanionSpeechClientKind = TEXT("http-sidecar"); + + UPROPERTY() + TObjectPtr ActiveCompanionSpeechClientObject; + UPROPERTY() bool bHasActiveRun = false; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h index f7b8536..7ca1f14 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h @@ -1890,6 +1890,48 @@ struct FHyperTwistTrainingRecognitionSessionState FHyperTwistVisionReconstructionSession ActiveReconstructionSession; }; +USTRUCT(BlueprintType) +struct FHyperTwistTrainingCompanionSpeechSessionState +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSessionOpen = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bUsingProviderBackedClient = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bHasTranscriptResult = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ActiveSessionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString OpenedAtUtc; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastUpdatedAtUtc; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastError; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SubmittedUtteranceCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 FinalTranscriptCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechSessionConfig SessionConfig; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechServiceHealth ServiceHealth; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechTranscriptResult LastTranscriptResult; +}; + USTRUCT(BlueprintType) struct FHyperTwistTrainingRunStepResult { diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp new file mode 100644 index 0000000..0702404 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp @@ -0,0 +1,199 @@ +// Copyright HyperTwist, Inc. All Rights Reserved. + +#include "Misc/AutomationTest.h" + +#include "Engine/GameInstance.h" +#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h" +#include "UObject/UnrealType.h" + +#if WITH_AUTOMATION_TESTS + +namespace HyperTwistWhisperCppPhase6RETestInternal +{ + FHyperTwistTrainingDeck MakeSpeechDeck() + { + FHyperTwistTrainingDeck Deck; + Deck.DeckId = TEXT("phase6r-e/whispercpp-speech"); + Deck.Title = TEXT("Phase 6R-E Whisper Speech"); + Deck.DeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + + FHyperTwistTrainingCase TrainingCase; + TrainingCase.CaseId = TEXT("phase6r-e-case"); + TrainingCase.PuzzleId = TEXT("cube/3x3x3"); + TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence; + TrainingCase.PromptLabel = TEXT("Phase 6R-E Coach Speech"); + TrainingCase.AllowedDeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + Deck.Cases = {TrainingCase}; + return Deck; + } + + void ForceMockSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem) + { + if (TrainingSubsystem == nullptr) + { + return; + } + + if (FStrProperty* SpeechClientKindProperty = FindFProperty( + UHyperTwistTrainingSubsystem::StaticClass(), + TEXT("CompanionSpeechClientKind") + )) + { + SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, TEXT("mock")); + } + } + + UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId) + { + UGameInstance* GameInstance = NewObject(GetTransientPackage()); + if (GameInstance == nullptr) + { + return nullptr; + } + + UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject(GameInstance); + if (TrainingSubsystem == nullptr) + { + return nullptr; + } + + ForceMockSpeechClient(TrainingSubsystem); + const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck( + MakeSpeechDeck(), + TEXT("phase6r-e-user"), + SessionId, + EHyperTwistTrainingDeliveryMode::CoachReviewed + ); + return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr; + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RESessionConfigTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.E.SessionConfig", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RESessionConfigTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-config-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue( + TEXT("The companion speech session must open for a coach-reviewed run."), + TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError) + ); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + TestEqual( + TEXT("The speech session must anchor on the existing listening contract."), + SessionState.SessionConfig.ListeningContractId, + TEXT("listening-threshold-lifecycle") + ); + TestEqual( + TEXT("The speech session must preserve whisper.cpp mono PCM routing."), + SessionState.SessionConfig.AudioEncoding, + TEXT("pcm-f32le-16khz-mono") + ); + TestEqual( + TEXT("The speech session must keep the bounded coach command grammar profile."), + SessionState.SessionConfig.GrammarProfileId, + TEXT("coach-command-grammar-v1") + ); + TestTrue(TEXT("The speech session must keep VAD enabled."), SessionState.SessionConfig.bEnableVad); + TestTrue( + TEXT("The bounded route must preserve repeat-case as a command hint."), + SessionState.SessionConfig.CommandHints.Contains(TEXT("repeat case")) + ); + TestTrue(TEXT("The mock speech health must report ready."), SessionState.ServiceHealth.bReady); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RETranscriptResultTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.E.TranscriptResult", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RETranscriptResultTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-transcript-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FHyperTwistSpeechUtteranceEnvelope Utterance; + Utterance.AudioRef = TEXT("mock://coach/repeat-case"); + Utterance.SpeechStartMs = 120; + Utterance.SpeechEndMs = 860; + Utterance.SilenceGapMs = 200; + Utterance.SampleCount = 11840; + const FHyperTwistSpeechTranscriptResult Result = + TrainingSubsystem->SubmitActiveCompanionSpeechUtterance(Utterance); + + TestTrue(TEXT("The transcript result must be structurally valid."), Result.IsStructurallyValid()); + TestEqual(TEXT("The bounded mock route must resolve the repeat-case transcript."), Result.TranscriptText, TEXT("repeat case")); + TestEqual(TEXT("The transcript result must keep one segment."), Result.Segments.Num(), 1); + if (Result.Segments.Num() > 0) + { + TestEqual(TEXT("The transcript segment must preserve the utterance start time."), Result.Segments[0].StartMs, 120); + TestEqual(TEXT("The transcript segment must preserve the utterance end time."), Result.Segments[0].EndMs, 860); + TestTrue(TEXT("The transcript segment must stay grammar-constrained."), Result.Segments[0].bGrammarConstrained); + } + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + TestEqual(TEXT("The active speech state must count one submitted utterance."), SessionState.SubmittedUtteranceCount, 1); + TestEqual(TEXT("The active speech state must count one final transcript."), SessionState.FinalTranscriptCount, 1); + TestEqual(TEXT("The active speech state must retain the last transcript text."), SessionState.LastTranscriptResult.TranscriptText, TEXT("repeat case")); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RECloseSessionTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.E.CloseSession", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RECloseSessionTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-close-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue(TEXT("The speech session must open cleanly before close."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + FString CloseError; + TestTrue(TEXT("The speech session must close cleanly."), TrainingSubsystem->CloseActiveCompanionSpeechSession(CloseError)); + TestTrue(TEXT("Closing the speech session must not report an error."), CloseError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + TestFalse(TEXT("The speech session must no longer report as open."), SessionState.bSessionOpen); + TestTrue(TEXT("The mock speech health should remain available after close."), SessionState.ServiceHealth.bReady); + TestEqual(TEXT("The mock speech provider label must stay stable."), SessionState.ServiceHealth.ProviderLabel, TEXT("mock")); + return true; +} + +#endif diff --git a/docs/REPO_LICENSE_TRACKING.md b/docs/REPO_LICENSE_TRACKING.md index fea471d..a821af5 100644 --- a/docs/REPO_LICENSE_TRACKING.md +++ b/docs/REPO_LICENSE_TRACKING.md @@ -1396,6 +1396,7 @@ Decision date: - `2026-04-24` - refreshed on `2026-05-13` +- implementation posture refreshed on `2026-05-21` Current licensing judgment: @@ -1420,6 +1421,9 @@ Approved working posture: - HyperTwist may use the codebase directly as the primary offline STT sidecar candidate under the `MIT` code posture - keep it bounded behind a speech-input seam - prefer command, dictation, and constrained coach-interaction use over broad “voice assistant platform” scope +- the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle +- keep model files, downloadable payloads, and future voice-asset review separate from the code-license judgment +- treat `SYSTRAN/faster-whisper` as the next complementary queue head for Python orchestration rather than widening `whisper.cpp` into the runtime core ### `SYSTRAN/faster-whisper` diff --git a/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md b/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md new file mode 100644 index 0000000..d33a01e --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md @@ -0,0 +1,159 @@ +# HyperTwist Phase 6R-E whisper.cpp speech transcript implementation packet + +Created on `2026-05-21` + +## Status + +- first-party HyperTwist packet +- bounded permissive `Phase 6R-E` implementation slice + +## Purpose + +This packet lands the first narrower bounded slice from the retained `ggml-org/whisper.cpp` +speech-input row. + +The landed slice is: + +- first-party speech transcript session boundary above the existing companion listening lifecycle + +It is not: + +- a full `whisper.cpp` row transplant +- a live microphone capture packet +- a model checkout or payload-shipping packet +- a `SYSTRAN/faster-whisper` Python orchestration packet +- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet +- a broad voice-assistant platform packet + +## Current authority basis + +This implementation packet stands on: + +- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md` +- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md` +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md` + +The retained owner remains: + +- `ggml-org/whisper.cpp` + +The granted family remains bounded to: + +- speech session configuration +- VAD-aware utterance-envelope shaping +- transcript-segment and transcript-result structure +- service-health and sidecar-boundary shaping + +This packet lands only the first narrower family in that granted set. + +## Landed scope + +The current code now owns a retained speech transcript contract through: + +- retained recognition contract types for: + - speech VAD policy + - speech session configuration + - speech utterance envelope + - speech transcript segment + - speech transcript result + - speech service health +- companion speech session state in: + - `FHyperTwistTrainingCompanionSpeechSessionState` +- sample session-config, transcript-result, and service-health outputs in: + - `UHyperTwistContractLibrary` +- direct-donor speech client seam with: + - `UHyperTwistMockSpeechClient` + - `UHyperTwistHttpSpeechClient` +- active companion open / submit / close transcript-session handling in: + - `UHyperTwistTrainingSubsystem` +- default sidecar configuration in: + - `DefaultGame.ini` +- focused automation coverage in: + - `HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp` + +## Why this is still intentionally bounded + +This packet lands the first donor-backed transcript-session seam, but it does not widen into the +neighboring retained families. + +Still deferred: + +- live microphone capture shell +- device-permission shell +- downloadable model or payload shipping +- Python-side orchestration, batching, or hotword service work +- TTS or voice-output ownership +- broad voice-assistant scope + +## Validation + +Build validation: + +- `Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE` + +Focused automation validation: + +- `Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.E` + +Regression automation validation: + +- `Automation RunTests HyperTwist.Permissive.MagicTile.Phase6R.D` +- `Automation RunTests HyperTwist.Permissive.RubixCubeSolver.Phase6R.C` +- `Automation RunTests HyperTwist.Permissive.Qbr.Phase6R.B` +- `Automation RunTests HyperTwist.Permissive.Hyperspeedcube.Phase6R.A` +- `Automation RunTests HyperTwist.CleanRoom.CubeDesk` + +Expected covered tests: + +- `CloseSession` +- `SessionConfig` +- `TranscriptResult` +- `TopologyContract` +- `LoaderBoundary` +- `PackageChecklist` +- `CommitBuildsReconstructionSession` +- `RevisionLedger` +- `FinalizeBuildsClassicNet` +- `CalibrationSessionConfig` +- `PreviewObservation` +- `CommitObservation` +- `CatalogLookup` +- `SampleDefinition` +- `TrainingRunDefinition` +- existing `CubeDesk` clean-room regression suite + +## Queue effect + +This packet consumes the current `Phase 6R-E` implementation slice. + +`ggml-org/whisper.cpp` remains only partially incorporated: + +- landed now: + - speech transcript session boundary + - bounded VAD-aware session config + - utterance-envelope shape + - transcript segment and transcript result + - speech-service health and sidecar boundary +- still deferred: + - live microphone capture shell + - downloadable model or payload shipping + - Python orchestration / batching / hotword service layer + - TTS or voice-output ownership + - broad voice-assistant scope + +The next clean move is not another immediate `whisper.cpp` widening by default. + +The next queue head should be: + +- a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` preparation/control pass + +Keep the future sequencing guards visible: + +- speech-input / voice sidecar set + - keep code-license judgments separate from model, voice, and payload-license review +- `SYSTRAN/faster-whisper` + - keep it complementary to `whisper.cpp` and bounded to Python transcription-service + orchestration rather than runtime-core ownership +- `rhasspy/piper` and `coqui-ai/TTS` + - keep future voice-asset review separate from the code-license judgment diff --git a/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md b/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md new file mode 100644 index 0000000..9a43219 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md @@ -0,0 +1,182 @@ +# HyperTwist Phase 6R-E whisper.cpp speech transcript preparation packet + +Created on `2026-05-21` + +## Status + +- historical same-day preparation authority +- bounded post-Phase-`6R-D` preparation slice +- the first bounded implementation slice now lands separately under: + - `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md` + +## Purpose + +This packet freezes the next widening order after the landed `Phase 6R-D` `roice3/MagicTile` +tiling-topology slice. + +The open task was: + +- define the first bounded source-backed widening packet for `ggml-org/whisper.cpp` as the + retained `Phase 6R-E` speech transcript session boundary + +It is not: + +- a full `whisper.cpp` row transplant +- a live microphone capture or device-permission packet +- a model checkout or downloadable payload-shipping packet +- a `SYSTRAN/faster-whisper` Python orchestration packet +- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet +- a broad “voice assistant platform” packet + +## Current authority basis + +This preparation packet stands on already-closed authority: + +- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md` +- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md` +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/arch/HYPERTWIST_PHASE6R_D_MAGICTILE_TILING_TOPOLOGY_IMPLEMENTATION_PACKET_2026-05-21.md` + +The key accepted routing facts are: + +- `ggml-org/whisper.cpp` is the retained offline STT donor immediately after the landed `6R-D` + slice +- earliest widening route is `Phase 6R / Packet 6R-E` +- the retained donor value there is: + - speech session configuration + - bounded VAD policy + - utterance-envelope shaping + - transcript-segment and transcript-result structure + - service-health / sidecar-boundary shaping +- ownership denied there is: + - do not let the row absorb microphone capture or permission shell ownership + - do not let the row absorb downloadable model, payload, or voice-asset review + - do not widen into Python orchestration that belongs with `SYSTRAN/faster-whisper` + - do not widen into TTS or broad voice-output ownership + - do not widen into a broad assistant platform + +## Why this was the next packet + +The earlier retained queue heads were already landed: + +- `Phase 6R-A` `HactarCE/Hyperspeedcube` +- `Phase 6R-B` `kkoomen/qbr` +- `Phase 6R-C` `vivaansinghvi07/rubix-cube-solver` +- `Phase 6R-D` `roice3/MagicTile` + +That advanced the live queue to: + +- `Phase 6R-E` `ggml-org/whisper.cpp` + +with the later adjacent retained rows ordered behind it: + +- `Phase 6R-F` `SYSTRAN/faster-whisper` +- `rhasspy/piper` +- `coqui-ai/TTS` + +## Required result + +The source-backed control pass for this packet is now complete. + +The first actual `6R-E` implementation packet should: + +- use the retained `0R-A` and `2R-A` authority surfaces plus the inspected `whisper.cpp` source + basis +- define one bounded retained slice from `whisper.cpp` +- keep the slice inside the accepted speech-input seam +- widen only the first transcript-session family that can stand on its own without dragging in + capture-shell, model-shipping, or voice-output scope +- explicitly state which neighboring retained rows stay closed in that packet + +## Source-backed retained basis + +The queue-head decision is now source-backed rather than README-only. + +Inspected retained donor basis: + +- `README.md` +- `include/whisper.h` +- `examples/server/server.cpp` + +Inspected first-party receiving basis: + +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp` + +## Narrowed 6R-E slice decision + +The first widening slice was fixed as: + +1. speech transcript session boundary above the existing listening lifecycle + +not as the broader speech-input or voice platform. + +That narrowed slice covers: + +- first-party speech VAD policy +- first-party speech session configuration +- first-party utterance envelope +- first-party transcript segment and transcript result +- first-party speech-service health contract +- first-party companion speech session state +- contract-library sample session config and transcript outputs +- direct-donor speech client seam with mock and HTTP-sidecar implementations +- active companion open / submit / close transcript-session handling +- focused automation coverage + +## Deferred neighboring families + +The first `6R-E` implementation packet must keep these capability families closed: + +- live microphone capture shell +- device-permission shell +- downloadable model or payload shipping +- Python orchestration, batching, or hotword service-layer work that belongs with + `SYSTRAN/faster-whisper` +- TTS or voice-output ownership +- broad “voice assistant platform” scope + +Why this slice is first: + +- `whisper.cpp` exposes the strongest narrow retained seam at bounded offline transcription with + VAD-aware session and transcript shape +- current first-party code already has a companion listening lifecycle but lacked an owned + transcript-session contract and sidecar boundary +- keeping model, payload, and voice decisions outside this packet preserves swapability and cleaner + legal review + +## Out of scope for the first 6R-E packet + +- live microphone capture +- downloadable model checkout or packaging +- TTS / voice output +- `faster-whisper` Python service orchestration +- `piper` or `coqui-ai/TTS` +- broad assistant or narration platform work + +## Acceptance criteria + +- the packet records why the live queue advanced from landed `6R-D` to `6R-E` +- the packet records that the first `6R-E` widening slice is the speech transcript session + boundary only +- the packet states exact out-of-scope families for the first `6R-E` pass +- the packet keeps the speech-lane legal sequencing guard visible: + - keep code-license judgments separate from model, voice, and payload-license review +- the packet leaves Python orchestration and voice-output rows deferred behind the landed slice + +## Validation checklist + +1. confirm the landed `Phase 6R-D` packet is the consumed prior queue head +2. confirm the retained `whisper.cpp` seam narrows cleanly to the transcript-session contract and + sidecar boundary +3. confirm the next implementation packet is framed as bounded source-backed widening rather than a + broad speech-platform transplant + +That is the packet. diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md index 2f8ca5a..214509f 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md @@ -44,8 +44,14 @@ Status update on `2026-05-21`: packet is now landed in current code - `roice3/MagicTile` remains partially incorporated; transform-aware macro remapping, broader non-Euclidean interaction, and host-shell ownership stay deferred -- the current next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar - preparation/control pass, starting with `ggml-org/whisper.cpp` +- the generic source-backed `Phase 6R-E` speech-input / voice sidecar control pass is now + consumed +- the bounded permissive `Phase 6R-E` `ggml-org/whisper.cpp` speech transcript session packet is + now landed in current code +- `ggml-org/whisper.cpp` remains partially incorporated; live microphone capture, model/payload + shipping, Python orchestration, and voice-output ownership stay deferred +- the current next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` + preparation/control pass - the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than waiting on another first-party packet @@ -73,8 +79,8 @@ Donor-strength rule: - route difficulty changes how retained value may enter the product, not whether it may win technically - current shallow-eval set: `75` repos -- currently verified live/implemented in checked Unreal surfaces: `28` -- permissive live lanes: `17` +- currently verified live/implemented in checked Unreal surfaces: `29` +- permissive live lanes: `18` - boundary-sensitive live lanes: `6` - restrictive live lanes: `5` - the restrictive landed lanes are: @@ -107,8 +113,8 @@ Current routing truth: - active non-live implementation-board rows: `35` - retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9` -The next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar -preparation/control pass, starting with `ggml-org/whisper.cpp`. +The next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` +preparation/control pass. Queue interpretation after that packet: @@ -153,8 +159,19 @@ Queue interpretation after that packet: - transform-aware macro remapping - broad non-Euclidean interaction shell - WinForms/OpenTK host shell -- after the `Phase 6R-D` implementation packet, keep the next queue shape explicit: - - `ggml-org/whisper.cpp` +- `ggml-org/whisper.cpp` remains a partially landed row rather than a closed row: + - landed: + - speech transcript session boundary + - VAD-aware session config and utterance envelope + - transcript segment and transcript result + - sidecar health and speech client boundary + - still deferred: + - live microphone capture shell + - downloadable model or payload shipping + - Python orchestration / batching / hotword service layer + - TTS or voice-output ownership + - broad assistant-platform scope +- after the `Phase 6R-E` implementation packet, keep the next queue shape explicit: - `SYSTRAN/faster-whisper` - `rhasspy/piper` - `coqui-ai/TTS`