Implement Phase 6R-E speech transcript session boundary

This commit is contained in:
axiomlogicnexus 2026-05-21 06:19:02 +02:00
parent e0490fcf7d
commit d0fc383801
15 changed files with 1905 additions and 8 deletions

View file

@ -15,3 +15,12 @@ FinalizeSessionPath=/vision/session/finalize
CloseSessionPath=/vision/session/close
HealthPath=/vision/health
RequestTimeoutSeconds=5.0
[/Script/UnrealHyperTwist.HyperTwistHttpSpeechClient]
ProviderLabel=local-http-sidecar
ServiceBaseUrl=http://127.0.0.1:8766
OpenSessionPath=/speech/session/open
TranscribeUtterancePath=/speech/utterance
CloseSessionPath=/speech/session/close
HealthPath=/speech/health
RequestTimeoutSeconds=5.0

View file

@ -612,6 +612,72 @@ FHyperTwistVisionServiceHealth UHyperTwistContractLibrary::MakeMockVisionService
return Health;
}
FHyperTwistSpeechSessionConfig UHyperTwistContractLibrary::MakeSampleSpeechSessionConfig()
{
FHyperTwistSpeechSessionConfig Config;
Config.SessionId = TEXT("speech_01");
Config.ListeningContractId = TEXT("listening-threshold-lifecycle");
Config.InputRouteId = TEXT("queue/listening");
Config.AudioEncoding = TEXT("pcm-f32le-16khz-mono");
Config.SampleRateHz = 16000;
Config.ChannelCount = 1;
Config.LanguageMode = TEXT("auto");
Config.TaskKind = TEXT("transcribe");
Config.GrammarProfileId = TEXT("coach-command-grammar-v1");
Config.bEnableVad = true;
Config.bEnableWordTimestamps = false;
Config.VadPolicy.Threshold = 0.5f;
Config.VadPolicy.MinSpeechDurationMs = 250;
Config.VadPolicy.MinSilenceDurationMs = 200;
Config.VadPolicy.MaxSpeechDurationMs = 12000;
Config.VadPolicy.SpeechPadMs = 100;
Config.CommandHints = {
TEXT("ready coach"),
TEXT("repeat case"),
TEXT("next case")
};
return Config;
}
FHyperTwistSpeechTranscriptResult UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult()
{
FHyperTwistSpeechTranscriptResult Result;
Result.SessionId = TEXT("speech_01");
Result.UtteranceId = TEXT("speech_01_utterance_001");
Result.LanguageCode = TEXT("en");
Result.TaskKind = TEXT("transcribe");
Result.GrammarProfileId = TEXT("coach-command-grammar-v1");
Result.bIsFinal = true;
Result.TranscriptText = TEXT("ready coach");
FHyperTwistSpeechTranscriptSegment Segment;
Segment.SegmentOrdinal = 0;
Segment.Text = Result.TranscriptText;
Segment.StartMs = 120;
Segment.EndMs = 840;
Segment.Confidence = 0.94f;
Segment.NoSpeechProbability = 0.04f;
Segment.bGrammarConstrained = true;
Result.Segments = {Segment};
return Result;
}
FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechServiceHealth()
{
FHyperTwistSpeechServiceHealth Health;
Health.ProviderLabel = TEXT("mock");
Health.ServiceVersion = TEXT("mock-speech/v1");
Health.ServiceEndpoint = TEXT("in-process://mock");
Health.Capabilities = {
TEXT("transcribe"),
TEXT("vadSegments"),
TEXT("grammarHints"),
TEXT("sessioned-mock-client")
};
Health.bReady = true;
return Health;
}
FHyperTwistContentPack UHyperTwistContractLibrary::MakeSampleAlgTrainerContentPack()
{
return UHyperTwistTrainingCatalogLibrary::MakeAlgTrainerStarterContentPack();

View file

@ -0,0 +1,314 @@
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
#include "Dom/JsonObject.h"
#include "Http.h"
#include "HttpManager.h"
#include "HttpModule.h"
#include "Interfaces/IHttpRequest.h"
#include "Interfaces/IHttpResponse.h"
#include "JsonObjectConverter.h"
#include "Serialization/JsonSerializer.h"
#include "Serialization/JsonWriter.h"
namespace HyperTwistHttpSpeechClientInternal
{
template <typename TStruct>
bool SerializeStruct(const TStruct& Value, FString& OutJson)
{
return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0);
}
template <typename TStruct>
bool DeserializeStruct(const FString& Json, TStruct& OutValue)
{
return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0);
}
FString MakeSessionIdJson(const FString& SessionId)
{
FString Json;
const TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
Root->SetStringField(TEXT("sessionId"), SessionId);
const TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&Json);
FJsonSerializer::Serialize(Root, Writer);
return Json;
}
FString BuildUrl(const FString& BaseUrl, const FString& Path)
{
if (BaseUrl.IsEmpty())
{
return FString();
}
const FString TrimmedBase = BaseUrl.EndsWith(TEXT("/")) ? BaseUrl.LeftChop(1) : BaseUrl;
if (Path.IsEmpty())
{
return TrimmedBase;
}
const FString TrimmedPath = Path.StartsWith(TEXT("/")) ? Path.RightChop(1) : Path;
return FString::Printf(TEXT("%s/%s"), *TrimmedBase, *TrimmedPath);
}
struct FHttpJsonResponse
{
bool bCompleted = false;
bool bSucceeded = false;
int32 StatusCode = 0;
FString ResponseBody;
FString Error;
};
bool ExecuteJsonRequest(
const UHyperTwistHttpSpeechClient& Client,
const FString& Verb,
const FString& Url,
const FString& RequestJson,
FHttpJsonResponse& OutResponse)
{
OutResponse = FHttpJsonResponse();
if (Url.IsEmpty())
{
OutResponse.Error = TEXT("service-base-url-missing");
return false;
}
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
Request->SetURL(Url);
Request->SetVerb(Verb);
Request->SetHeader(TEXT("Accept"), TEXT("application/json"));
Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
if (!Client.AuthorizationToken.IsEmpty())
{
Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *Client.AuthorizationToken));
}
if (!RequestJson.IsEmpty())
{
Request->SetContentAsString(RequestJson);
}
Request->OnProcessRequestComplete().BindLambda(
[&OutResponse](FHttpRequestPtr, FHttpResponsePtr Response, bool bWasSuccessful)
{
OutResponse.bCompleted = true;
if (Response.IsValid())
{
OutResponse.StatusCode = Response->GetResponseCode();
OutResponse.ResponseBody = Response->GetContentAsString();
}
if (!bWasSuccessful)
{
OutResponse.Error = TEXT("request-failed");
return;
}
if (!Response.IsValid())
{
OutResponse.Error = TEXT("response-missing");
return;
}
if (!EHttpResponseCodes::IsOk(OutResponse.StatusCode))
{
OutResponse.Error = FString::Printf(TEXT("http-%d"), OutResponse.StatusCode);
return;
}
OutResponse.bSucceeded = true;
});
if (!Request->ProcessRequest())
{
OutResponse.Error = TEXT("request-start-failed");
return false;
}
const double TimeoutSeconds = FMath::Max(static_cast<double>(Client.RequestTimeoutSeconds), 0.1);
const double Deadline = FPlatformTime::Seconds() + TimeoutSeconds;
while (!OutResponse.bCompleted && FPlatformTime::Seconds() < Deadline)
{
FHttpModule::Get().GetHttpManager().Tick(0.01f);
FPlatformProcess::Sleep(0.01f);
}
if (!OutResponse.bCompleted)
{
Request->CancelRequest();
OutResponse.Error = TEXT("request-timeout");
return false;
}
return OutResponse.bSucceeded;
}
}
bool UHyperTwistHttpSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError)
{
OutError.Reset();
FString RequestJson;
if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Config, RequestJson))
{
OutError = TEXT("request-serialize-failed");
LastTransportError = OutError;
return false;
}
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, OpenSessionPath),
RequestJson,
Response))
{
OutError = Response.Error;
LastTransportError = OutError;
return false;
}
LastTransportError.Reset();
return true;
}
FHyperTwistSpeechTranscriptResult UHyperTwistHttpSpeechClient::TranscribeSpeechUtterance(
const FHyperTwistSpeechUtteranceEnvelope& Utterance
)
{
FHyperTwistSpeechTranscriptResult Result;
Result.SessionId = Utterance.SessionId;
Result.UtteranceId = Utterance.UtteranceId;
FString RequestJson;
if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Utterance, RequestJson))
{
LastTransportError = TEXT("request-serialize-failed");
Result.Warnings.Add(LastTransportError);
return Result;
}
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, TranscribeUtterancePath),
RequestJson,
Response))
{
LastTransportError = Response.Error;
Result.Warnings.Add(LastTransportError);
return Result;
}
if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Result))
{
LastTransportError = TEXT("invalid-provider-response");
Result = FHyperTwistSpeechTranscriptResult();
Result.SessionId = Utterance.SessionId;
Result.UtteranceId = Utterance.UtteranceId;
Result.Warnings.Add(LastTransportError);
return Result;
}
if (Result.SessionId.IsEmpty())
{
Result.SessionId = Utterance.SessionId;
}
if (Result.UtteranceId.IsEmpty())
{
Result.UtteranceId = Utterance.UtteranceId;
}
LastTransportError.Reset();
return Result;
}
bool UHyperTwistHttpSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError)
{
OutError.Reset();
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath),
HyperTwistHttpSpeechClientInternal::MakeSessionIdJson(SessionId),
Response))
{
OutError = Response.Error;
LastTransportError = OutError;
return false;
}
LastTransportError.Reset();
return true;
}
FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHealth() const
{
FHyperTwistSpeechServiceHealth Health;
Health.ProviderLabel = ProviderLabel;
Health.ServiceVersion = TEXT("provider-unavailable/v1");
Health.ServiceEndpoint = ServiceBaseUrl;
Health.bReady = false;
Health.Capabilities = {
TEXT("transport:http"),
TEXT("provider-backed-speech"),
TEXT("transcribe"),
TEXT("vadSegments"),
TEXT("grammarHints")
};
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
*this,
TEXT("GET"),
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, HealthPath),
FString(),
Response))
{
LastTransportError = Response.Error;
Health.LastError = LastTransportError;
return Health;
}
if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Health))
{
LastTransportError = TEXT("invalid-provider-response");
Health = FHyperTwistSpeechServiceHealth();
Health.ProviderLabel = ProviderLabel;
Health.ServiceVersion = TEXT("provider-unavailable/v1");
Health.ServiceEndpoint = ServiceBaseUrl;
Health.bReady = false;
Health.Capabilities = {
TEXT("transport:http"),
TEXT("provider-backed-speech"),
TEXT("transcribe"),
TEXT("vadSegments"),
TEXT("grammarHints")
};
Health.LastError = LastTransportError;
return Health;
}
if (Health.ProviderLabel.IsEmpty())
{
Health.ProviderLabel = ProviderLabel;
}
if (Health.ServiceEndpoint.IsEmpty())
{
Health.ServiceEndpoint = ServiceBaseUrl;
}
Health.Capabilities.AddUnique(TEXT("transport:http"));
Health.Capabilities.AddUnique(TEXT("provider-backed-speech"));
LastTransportError.Reset();
Health.LastError.Reset();
return Health;
}

View file

@ -0,0 +1,119 @@
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
#include "HyperTwistBootstrap/HyperTwistContractLibrary.h"
namespace HyperTwistMockSpeechClientInternal
{
FString ResolveTranscriptText(
const FHyperTwistSpeechSessionConfig* SessionConfig,
const FHyperTwistSpeechUtteranceEnvelope& Utterance
)
{
const FString NormalizedAudioRef = Utterance.AudioRef.ToLower();
if (NormalizedAudioRef.Contains(TEXT("repeat-case")))
{
return TEXT("repeat case");
}
if (NormalizedAudioRef.Contains(TEXT("next-case")))
{
return TEXT("next case");
}
if (NormalizedAudioRef.Contains(TEXT("ready-coach")))
{
return TEXT("ready coach");
}
if (Utterance.LocalCommandHints.Num() > 0 && !Utterance.LocalCommandHints[0].IsEmpty())
{
return Utterance.LocalCommandHints[0];
}
if (SessionConfig != nullptr && SessionConfig->CommandHints.Num() > 0 && !SessionConfig->CommandHints[0].IsEmpty())
{
return SessionConfig->CommandHints[0];
}
return TEXT("ready coach");
}
}
bool UHyperTwistMockSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError)
{
OutError.Reset();
if (!Config.IsStructurallyValid())
{
OutError = TEXT("speech-session-config-invalid");
return false;
}
SessionConfigs.Add(Config.SessionId, Config);
SessionUtteranceCounts.Add(Config.SessionId, 0);
return true;
}
FHyperTwistSpeechTranscriptResult UHyperTwistMockSpeechClient::TranscribeSpeechUtterance(
const FHyperTwistSpeechUtteranceEnvelope& Utterance
)
{
FHyperTwistSpeechTranscriptResult Result = UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult();
Result.SessionId = Utterance.SessionId;
Result.UtteranceId = Utterance.UtteranceId;
const FHyperTwistSpeechSessionConfig* SessionConfig = SessionConfigs.Find(Utterance.SessionId);
if (SessionConfig == nullptr)
{
Result.Warnings.Add(TEXT("session-not-open"));
return Result;
}
const FString TranscriptText =
HyperTwistMockSpeechClientInternal::ResolveTranscriptText(SessionConfig, Utterance);
SessionUtteranceCounts.Add(Utterance.SessionId, SessionUtteranceCounts.FindRef(Utterance.SessionId) + 1);
Result.LanguageCode = SessionConfig->LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase)
? TEXT("en")
: SessionConfig->LanguageMode;
Result.TaskKind = SessionConfig->TaskKind;
Result.GrammarProfileId = SessionConfig->GrammarProfileId;
Result.bIsFinal = true;
Result.TranscriptText = TranscriptText;
Result.Segments.Reset();
FHyperTwistSpeechTranscriptSegment Segment;
Segment.SegmentOrdinal = 0;
Segment.Text = TranscriptText;
Segment.StartMs = FMath::Max(0, Utterance.SpeechStartMs);
Segment.EndMs = FMath::Max(Segment.StartMs, Utterance.SpeechEndMs);
Segment.Confidence = 0.93f;
Segment.NoSpeechProbability = 0.05f;
Segment.bGrammarConstrained = !SessionConfig->GrammarProfileId.IsEmpty();
Result.Segments.Add(Segment);
return Result;
}
bool UHyperTwistMockSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError)
{
OutError.Reset();
if (!SessionConfigs.Contains(SessionId))
{
OutError = TEXT("session-not-open");
return false;
}
SessionConfigs.Remove(SessionId);
SessionUtteranceCounts.Remove(SessionId);
return true;
}
FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHealth() const
{
FHyperTwistSpeechServiceHealth Health = UHyperTwistContractLibrary::MakeMockSpeechServiceHealth();
Health.ProviderLabel = TEXT("mock");
Health.ServiceEndpoint = TEXT("in-process://mock");
Health.Capabilities.AddUnique(TEXT("offline-transcribe"));
return Health;
}

View file

@ -1,6 +1,7 @@
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
#include "HyperTwistRecognition/HyperTwistVisionClient.h"
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
#include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h"
#include "HyperTwistReplay/HyperTwistReplayLibrary.h"
#include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h"
@ -11,6 +12,7 @@
#include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingPublicationLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h"
#include "HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h"
#include "JsonObjectConverter.h"
namespace HyperTwistTrainingSubsystemInternal
@ -1855,17 +1857,35 @@ namespace HyperTwistTrainingSubsystemInternal
: nullptr;
}
IHyperTwistSpeechClient* ResolveSpeechClient(UObject* Candidate)
{
return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistSpeechClient::StaticClass())
? Cast<IHyperTwistSpeechClient>(Candidate)
: nullptr;
}
FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
{
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
}
FString BuildSpeechSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
{
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
return FString::Printf(TEXT("%s_speech_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
}
FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal)
{
return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal));
}
FString BuildSpeechUtteranceId(const FString& SessionId, const int32 UtteranceOrdinal)
{
return FString::Printf(TEXT("%s_utterance_%03d"), *SessionId, FMath::Max(1, UtteranceOrdinal));
}
bool IsQbrClassicCubeRoute(const FHyperTwistVisionSessionConfig& SessionConfig)
{
return SessionConfig.ExpectedPuzzleFamily == EHyperTwistPuzzleFamily::ClassicCube
@ -3609,6 +3629,11 @@ FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActi
return ActiveRecognitionSessionState;
}
FHyperTwistTrainingCompanionSpeechSessionState UHyperTwistTrainingSubsystem::GetActiveCompanionSpeechSessionState() const
{
return ActiveCompanionSpeechSessionState;
}
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const
{
return ActiveCoachSignals;
@ -5487,7 +5512,13 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
if (ActiveCompanionSpeechSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveCompanionSpeechSession(CloseError);
}
ResetRecognitionSessionState();
ResetCompanionSpeechSessionState();
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus();
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
@ -6360,6 +6391,16 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
bAbortSession ? TEXT("training-run-aborted") : TEXT("training-run-completed")
);
}
if (ActiveRecognitionSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
if (ActiveCompanionSpeechSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveCompanionSpeechSession(CloseError);
}
ResetActiveLiveTimerState();
@ -6418,6 +6459,7 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
}
}
ResetRecognitionSessionState();
ResetCompanionSpeechSessionState();
RefreshRepositoryViews();
return ActiveRunState.Session;
}
@ -6429,6 +6471,11 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun()
FString CloseError;
CloseActiveRecognitionSession(CloseError);
}
if (ActiveCompanionSpeechSessionState.bSessionOpen)
{
FString CloseError;
CloseActiveCompanionSpeechSession(CloseError);
}
if (HasActiveSmartDeviceSession())
{
StopActiveSmartDeviceSession(TEXT("training-run-cleared"));
@ -6466,6 +6513,7 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun()
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
bHasActiveRun = false;
ResetRecognitionSessionState();
ResetCompanionSpeechSessionState();
RefreshRepositoryViews();
}
@ -7043,6 +7091,226 @@ bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutErr
return true;
}
bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& OutError)
{
OutError.Reset();
if (!HasActiveRun())
{
OutError = TEXT("no-active-training-run");
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
ResolveCompanionSpeechClientObject()
);
if (SpeechClient == nullptr)
{
OutError = TEXT("speech-client-unavailable");
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
const FHyperTwistSpeechSessionConfig SessionConfig = BuildActiveCompanionSpeechSessionConfig();
if (SessionConfig.SessionId.IsEmpty())
{
OutError = TEXT("speech-session-id-unavailable");
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
if (ActiveCompanionSpeechSessionState.bSessionOpen
&& ActiveCompanionSpeechSessionState.ActiveSessionId == SessionConfig.SessionId)
{
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
ActiveCompanionSpeechSessionState.LastError.Reset();
ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset();
return true;
}
if (ActiveCompanionSpeechSessionState.bSessionOpen
&& ActiveCompanionSpeechSessionState.ActiveSessionId != SessionConfig.SessionId)
{
FString CloseError;
SpeechClient->CloseSpeechSession(ActiveCompanionSpeechSessionState.ActiveSessionId, CloseError);
}
if (!SpeechClient->OpenSpeechSession(SessionConfig, OutError))
{
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveCompanionSpeechSessionState.bSessionOpen = true;
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient =
ActiveCompanionSpeechClientObject != nullptr
&& ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>();
ActiveCompanionSpeechSessionState.ActiveSessionId = SessionConfig.SessionId;
ActiveCompanionSpeechSessionState.OpenedAtUtc = OpenedAtUtc;
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = OpenedAtUtc;
ActiveCompanionSpeechSessionState.LastError.Reset();
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
ActiveCompanionSpeechSessionState.bHasTranscriptResult = false;
ActiveCompanionSpeechSessionState.LastTranscriptResult = FHyperTwistSpeechTranscriptResult();
RefreshCompanionSpeechServiceHealth();
return true;
}
FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveCompanionSpeechUtterance(
const FHyperTwistSpeechUtteranceEnvelope& Utterance
)
{
FHyperTwistSpeechTranscriptResult Result;
if (!HasActiveRun())
{
Result.Warnings.Add(TEXT("no-active-training-run"));
return Result;
}
FString OpenError;
if (!OpenActiveCompanionSpeechSession(OpenError))
{
Result.SessionId = !Utterance.SessionId.IsEmpty()
? Utterance.SessionId
: BuildActiveCompanionSpeechSessionConfig().SessionId;
Result.UtteranceId = Utterance.UtteranceId;
Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("speech-session-open-failed") : OpenError);
return Result;
}
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
ResolveCompanionSpeechClientObject()
);
if (SpeechClient == nullptr)
{
Result.Warnings.Add(TEXT("speech-client-unavailable"));
ActiveCompanionSpeechSessionState.LastError = Result.Warnings[0];
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError;
return Result;
}
FHyperTwistSpeechUtteranceEnvelope NormalizedUtterance = Utterance;
NormalizedUtterance.SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId;
if (NormalizedUtterance.UtteranceId.IsEmpty())
{
NormalizedUtterance.UtteranceId = HyperTwistTrainingSubsystemInternal::BuildSpeechUtteranceId(
NormalizedUtterance.SessionId,
ActiveCompanionSpeechSessionState.SubmittedUtteranceCount + 1
);
}
if (NormalizedUtterance.CapturedAtUtc.IsEmpty())
{
NormalizedUtterance.CapturedAtUtc = FDateTime::UtcNow().ToIso8601();
}
if (NormalizedUtterance.SampleRateHz <= 0)
{
NormalizedUtterance.SampleRateHz = ActiveCompanionSpeechSessionState.SessionConfig.SampleRateHz;
}
if (NormalizedUtterance.ChannelCount <= 0)
{
NormalizedUtterance.ChannelCount = ActiveCompanionSpeechSessionState.SessionConfig.ChannelCount;
}
if (NormalizedUtterance.LocalCommandHints.Num() <= 0)
{
NormalizedUtterance.LocalCommandHints = ActiveCompanionSpeechSessionState.SessionConfig.CommandHints;
}
NormalizedUtterance.SilenceGapMs = FMath::Max(0, NormalizedUtterance.SilenceGapMs);
NormalizedUtterance.SpeechStartMs = FMath::Max(0, NormalizedUtterance.SpeechStartMs);
NormalizedUtterance.SpeechEndMs = FMath::Max(
NormalizedUtterance.SpeechStartMs,
NormalizedUtterance.SpeechEndMs
);
Result = SpeechClient->TranscribeSpeechUtterance(NormalizedUtterance);
if (Result.SessionId.IsEmpty())
{
Result.SessionId = NormalizedUtterance.SessionId;
}
if (Result.UtteranceId.IsEmpty())
{
Result.UtteranceId = NormalizedUtterance.UtteranceId;
}
if (Result.TaskKind.IsEmpty())
{
Result.TaskKind = ActiveCompanionSpeechSessionState.SessionConfig.TaskKind;
}
if (Result.GrammarProfileId.IsEmpty())
{
Result.GrammarProfileId = ActiveCompanionSpeechSessionState.SessionConfig.GrammarProfileId;
}
if (Result.TranscriptText.IsEmpty() && Result.Segments.Num() > 0)
{
TArray<FString> SegmentTexts;
for (const FHyperTwistSpeechTranscriptSegment& Segment : Result.Segments)
{
if (!Segment.Text.IsEmpty())
{
SegmentTexts.Add(Segment.Text);
}
}
Result.TranscriptText = FString::Join(SegmentTexts, TEXT(" "));
}
++ActiveCompanionSpeechSessionState.SubmittedUtteranceCount;
if (Result.bIsFinal && Result.IsStructurallyValid())
{
++ActiveCompanionSpeechSessionState.FinalTranscriptCount;
}
ActiveCompanionSpeechSessionState.bHasTranscriptResult = true;
ActiveCompanionSpeechSessionState.LastTranscriptResult = Result;
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveCompanionSpeechSessionState.LastError =
Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString();
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError;
return Result;
}
bool UHyperTwistTrainingSubsystem::CloseActiveCompanionSpeechSession(FString& OutError)
{
OutError.Reset();
if (!ActiveCompanionSpeechSessionState.bSessionOpen || ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty())
{
RefreshCompanionSpeechServiceHealth();
return true;
}
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
ResolveCompanionSpeechClientObject()
);
if (SpeechClient == nullptr)
{
OutError = TEXT("speech-client-unavailable");
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
const FString SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId;
const bool bClosed = SpeechClient->CloseSpeechSession(SessionId, OutError);
if (!bClosed && OutError != TEXT("session-not-open"))
{
ActiveCompanionSpeechSessionState.LastError = OutError;
RefreshCompanionSpeechServiceHealth();
return false;
}
ActiveCompanionSpeechSessionState.bSessionOpen = false;
ActiveCompanionSpeechSessionState.ActiveSessionId.Reset();
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
ActiveCompanionSpeechSessionState.LastError.Reset();
RefreshCompanionSpeechServiceHealth();
return true;
}
void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
{
ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState();
@ -7410,6 +7678,74 @@ FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognit
return SessionConfig;
}
FHyperTwistSpeechSessionConfig UHyperTwistTrainingSubsystem::BuildActiveCompanionSpeechSessionConfig() const
{
FHyperTwistSpeechSessionConfig SessionConfig = ActiveCompanionSpeechSessionState.SessionConfig;
if (!HasActiveRun())
{
return SessionConfig;
}
const FHyperTwistTrainingCompanionReferenceBundle CompanionBundle =
UHyperTwistTrainingRuntimeLibrary::GetBundledEmbodiedCompanionReferenceBundle();
SessionConfig.SessionId = !ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty()
? ActiveCompanionSpeechSessionState.ActiveSessionId
: HyperTwistTrainingSubsystemInternal::BuildSpeechSessionId(ActiveRunState.Session, 1);
SessionConfig.ListeningContractId = !CompanionBundle.PrimaryListeningContractId.IsEmpty()
? CompanionBundle.PrimaryListeningContractId
: TEXT("listening-threshold-lifecycle");
if (SessionConfig.InputRouteId.IsEmpty())
{
SessionConfig.InputRouteId = TEXT("queue/listening");
}
if (SessionConfig.AudioEncoding.IsEmpty())
{
SessionConfig.AudioEncoding = TEXT("pcm-f32le-16khz-mono");
}
if (SessionConfig.SampleRateHz <= 0)
{
SessionConfig.SampleRateHz = 16000;
}
if (SessionConfig.ChannelCount <= 0)
{
SessionConfig.ChannelCount = 1;
}
if (SessionConfig.LanguageMode.IsEmpty())
{
SessionConfig.LanguageMode = TEXT("auto");
}
if (SessionConfig.TaskKind.IsEmpty())
{
SessionConfig.TaskKind = TEXT("transcribe");
}
if (SessionConfig.GrammarProfileId.IsEmpty())
{
SessionConfig.GrammarProfileId = TEXT("coach-command-grammar-v1");
}
SessionConfig.bEnableVad = true;
SessionConfig.bEnableWordTimestamps = false;
if (!SessionConfig.VadPolicy.IsStructurallyValid())
{
SessionConfig.VadPolicy.Threshold = 0.5f;
SessionConfig.VadPolicy.MinSpeechDurationMs = 250;
SessionConfig.VadPolicy.MinSilenceDurationMs = 200;
SessionConfig.VadPolicy.MaxSpeechDurationMs = 12000;
SessionConfig.VadPolicy.SpeechPadMs = 100;
}
if (SessionConfig.CommandHints.Num() <= 0)
{
SessionConfig.CommandHints = {
TEXT("ready coach"),
TEXT("repeat case"),
TEXT("next case"),
TEXT("pause coaching")
};
}
return SessionConfig;
}
void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
{
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
@ -7435,6 +7771,58 @@ void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth();
}
UObject* UHyperTwistTrainingSubsystem::ResolveCompanionSpeechClientObject()
{
LoadConfig();
const bool bUseMockClient = CompanionSpeechClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase);
const bool bNeedsNewClient = ActiveCompanionSpeechClientObject == nullptr
|| (bUseMockClient && !ActiveCompanionSpeechClientObject->IsA<UHyperTwistMockSpeechClient>())
|| (!bUseMockClient && !ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>());
if (bNeedsNewClient)
{
if (bUseMockClient)
{
ActiveCompanionSpeechClientObject = NewObject<UHyperTwistMockSpeechClient>(this);
}
else
{
UHyperTwistHttpSpeechClient* HttpSpeechClient = NewObject<UHyperTwistHttpSpeechClient>(this);
HttpSpeechClient->LoadConfig();
ActiveCompanionSpeechClientObject = HttpSpeechClient;
}
}
return ActiveCompanionSpeechClientObject;
}
void UHyperTwistTrainingSubsystem::RefreshCompanionSpeechServiceHealth()
{
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
ResolveCompanionSpeechClientObject()
);
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient =
ActiveCompanionSpeechClientObject != nullptr
&& ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>();
if (SpeechClient == nullptr)
{
ActiveCompanionSpeechSessionState.ServiceHealth = FHyperTwistSpeechServiceHealth();
ActiveCompanionSpeechSessionState.ServiceHealth.ProviderLabel =
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient
? TEXT("local-http-sidecar")
: TEXT("mock");
ActiveCompanionSpeechSessionState.ServiceHealth.ServiceEndpoint =
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient
? TEXT("http://127.0.0.1:8766")
: TEXT("in-process://mock");
ActiveCompanionSpeechSessionState.ServiceHealth.bReady = false;
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = TEXT("speech-client-unavailable");
return;
}
ActiveCompanionSpeechSessionState.ServiceHealth = SpeechClient->GetSpeechServiceHealth();
}
void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent(
const EHyperTwistReplayEventType EventType,
const FHyperTwistReplayRecognitionPayload& Payload,
@ -7492,6 +7880,12 @@ void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState()
RefreshRecognitionServiceHealth();
}
void UHyperTwistTrainingSubsystem::ResetCompanionSpeechSessionState()
{
ActiveCompanionSpeechSessionState = FHyperTwistTrainingCompanionSpeechSessionState();
RefreshCompanionSpeechServiceHealth();
}
bool UHyperTwistTrainingSubsystem::TryResolveRepositoryViewContext(
FString& OutUserId,
FString& OutDeckId,

View file

@ -57,6 +57,15 @@ public:
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
static FHyperTwistVisionServiceHealth MakeMockVisionServiceHealth();
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
static FHyperTwistSpeechSessionConfig MakeSampleSpeechSessionConfig();
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
static FHyperTwistSpeechTranscriptResult MakeMockSpeechTranscriptResult();
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
static FHyperTwistSpeechServiceHealth MakeMockSpeechServiceHealth();
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training")
static FHyperTwistContentPack MakeSampleAlgTrainerContentPack();

View file

@ -572,3 +572,263 @@ struct FHyperTwistVisionServiceHealth
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bReady = true;
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechVadPolicy
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
float Threshold = 0.5f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 MinSpeechDurationMs = 250;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 MinSilenceDurationMs = 200;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 MaxSpeechDurationMs = 12000;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SpeechPadMs = 100;
bool IsStructurallyValid() const
{
return Threshold > 0.0f
&& MinSpeechDurationMs >= 0
&& MinSilenceDurationMs >= 0
&& MaxSpeechDurationMs >= MinSpeechDurationMs
&& SpeechPadMs >= 0;
}
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechSessionConfig
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString SessionId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ListeningContractId = TEXT("listening-threshold-lifecycle");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString InputRouteId = TEXT("queue/listening");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString AudioEncoding = TEXT("pcm-f32le-16khz-mono");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SampleRateHz = 16000;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 ChannelCount = 1;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LanguageMode = TEXT("auto");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString TaskKind = TEXT("transcribe");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString GrammarProfileId = TEXT("coach-command-grammar-v1");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bEnableVad = true;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bEnableWordTimestamps = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistSpeechVadPolicy VadPolicy;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FString> CommandHints;
bool IsStructurallyValid() const
{
return !SessionId.IsEmpty()
&& !ListeningContractId.IsEmpty()
&& !InputRouteId.IsEmpty()
&& !AudioEncoding.IsEmpty()
&& SampleRateHz > 0
&& ChannelCount > 0
&& !TaskKind.IsEmpty()
&& VadPolicy.IsStructurallyValid();
}
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechUtteranceEnvelope
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString SessionId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString UtteranceId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString CapturedAtUtc;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString AudioRef;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SourceTimestampMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SampleCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SampleRateHz = 16000;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 ChannelCount = 1;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SpeechStartMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SpeechEndMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SilenceGapMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
float EnergyThreshold = 0.0f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FString> LocalCommandHints;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bSpeechCommitReady = true;
bool IsStructurallyValid() const
{
return !SessionId.IsEmpty()
&& !UtteranceId.IsEmpty()
&& SampleRateHz > 0
&& ChannelCount > 0
&& SpeechStartMs >= 0
&& SpeechEndMs >= SpeechStartMs
&& SilenceGapMs >= 0;
}
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechTranscriptSegment
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SegmentOrdinal = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString Text;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 StartMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 EndMs = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
float Confidence = 0.0f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
float NoSpeechProbability = 0.0f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bGrammarConstrained = false;
bool IsStructurallyValid() const
{
return SegmentOrdinal >= 0
&& !Text.IsEmpty()
&& StartMs >= 0
&& EndMs >= StartMs
&& Confidence >= 0.0f;
}
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechTranscriptResult
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString SessionId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString UtteranceId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LanguageCode = TEXT("en");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString TaskKind = TEXT("transcribe");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString GrammarProfileId = TEXT("coach-command-grammar-v1");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bIsFinal = true;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString TranscriptText;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FHyperTwistSpeechTranscriptSegment> Segments;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FString> Warnings;
bool IsStructurallyValid() const
{
if (SessionId.IsEmpty()
|| UtteranceId.IsEmpty()
|| TaskKind.IsEmpty()
|| (TranscriptText.IsEmpty() && Segments.Num() <= 0))
{
return false;
}
for (const FHyperTwistSpeechTranscriptSegment& Segment : Segments)
{
if (!Segment.IsStructurallyValid())
{
return false;
}
}
return true;
}
};
USTRUCT(BlueprintType)
struct FHyperTwistSpeechServiceHealth
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ProviderLabel = TEXT("mock");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ServiceVersion = TEXT("mock-speech/v1");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ServiceEndpoint;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FString> Capabilities;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastError;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bReady = true;
};

View file

@ -0,0 +1,96 @@
#pragma once
#include "CoreMinimal.h"
#include "UObject/Interface.h"
#include "UObject/Object.h"
#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h"
#include "HyperTwistSpeechClient.generated.h"
UINTERFACE(BlueprintType)
class UNREALHYPERTWIST_API UHyperTwistSpeechClient : public UInterface
{
GENERATED_BODY()
};
class UNREALHYPERTWIST_API IHyperTwistSpeechClient
{
GENERATED_BODY()
public:
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) = 0;
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) = 0;
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) = 0;
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const = 0;
};
UCLASS(BlueprintType)
class UNREALHYPERTWIST_API UHyperTwistMockSpeechClient : public UObject, public IHyperTwistSpeechClient
{
GENERATED_BODY()
public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override;
private:
UPROPERTY()
TMap<FString, FHyperTwistSpeechSessionConfig> SessionConfigs;
UPROPERTY()
TMap<FString, int32> SessionUtteranceCounts;
};
UCLASS(BlueprintType, Config = Game, DefaultConfig)
class UNREALHYPERTWIST_API UHyperTwistHttpSpeechClient : public UObject, public IHyperTwistSpeechClient
{
GENERATED_BODY()
public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString ProviderLabel = TEXT("local-http-sidecar");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString ServiceBaseUrl = TEXT("http://127.0.0.1:8766");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString OpenSessionPath = TEXT("/speech/session/open");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString TranscribeUtterancePath = TEXT("/speech/utterance");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString CloseSessionPath = TEXT("/speech/session/close");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString HealthPath = TEXT("/speech/health");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
float RequestTimeoutSeconds = 5.0f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString AuthorizationToken;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override;
private:
mutable FString LastTransportError;
};

View file

@ -176,6 +176,9 @@ public:
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition")
FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const;
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Speech")
FHyperTwistTrainingCompanionSpeechSessionState GetActiveCompanionSpeechSessionState() const;
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach")
TArray<FHyperTwistCoachSignal> GetActiveCoachSignals() const;
@ -685,6 +688,17 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
bool CloseActiveRecognitionSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
bool OpenActiveCompanionSpeechSession(FString& OutError);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
FHyperTwistSpeechTranscriptResult SubmitActiveCompanionSpeechUtterance(
const FHyperTwistSpeechUtteranceEnvelope& Utterance
);
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
bool CloseActiveCompanionSpeechSession(FString& OutError);
void SetActiveAlgJsSequence(const FHyperTwistAlgorithmSequence& Sequence);
bool TryGetActiveAlgJsSequence(FHyperTwistAlgorithmSequence& OutSequence) const;
@ -716,8 +730,11 @@ private:
const FString& NowUtc
) const;
FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const;
FHyperTwistSpeechSessionConfig BuildActiveCompanionSpeechSessionConfig() const;
UObject* ResolveRecognitionVisionClientObject();
UObject* ResolveCompanionSpeechClientObject();
void RefreshRecognitionServiceHealth();
void RefreshCompanionSpeechServiceHealth();
void AppendRecognitionReplayEvent(
EHyperTwistReplayEventType EventType,
const FHyperTwistReplayRecognitionPayload& Payload,
@ -725,6 +742,7 @@ private:
);
void SynchronizeRecognitionReplayMutation();
void ResetRecognitionSessionState();
void ResetCompanionSpeechSessionState();
TArray<FHyperTwistTrainingSessionTemplate> BuildTrainingSessionTemplatesForUser(
const FString& UserId,
const FString& ReferenceUtc
@ -820,6 +838,9 @@ private:
UPROPERTY()
FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState;
UPROPERTY()
FHyperTwistTrainingCompanionSpeechSessionState ActiveCompanionSpeechSessionState;
UPROPERTY()
TArray<FHyperTwistCoachSignal> ActiveCoachSignals;
@ -898,6 +919,12 @@ private:
UPROPERTY()
TObjectPtr<UObject> ActiveRecognitionVisionClientObject;
UPROPERTY(Config)
FString CompanionSpeechClientKind = TEXT("http-sidecar");
UPROPERTY()
TObjectPtr<UObject> ActiveCompanionSpeechClientObject;
UPROPERTY()
bool bHasActiveRun = false;

View file

@ -1890,6 +1890,48 @@ struct FHyperTwistTrainingRecognitionSessionState
FHyperTwistVisionReconstructionSession ActiveReconstructionSession;
};
USTRUCT(BlueprintType)
struct FHyperTwistTrainingCompanionSpeechSessionState
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bSessionOpen = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bUsingProviderBackedClient = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bHasTranscriptResult = false;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ActiveSessionId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString OpenedAtUtc;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastUpdatedAtUtc;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastError;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 SubmittedUtteranceCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
int32 FinalTranscriptCount = 0;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistSpeechSessionConfig SessionConfig;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistSpeechServiceHealth ServiceHealth;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FHyperTwistSpeechTranscriptResult LastTranscriptResult;
};
USTRUCT(BlueprintType)
struct FHyperTwistTrainingRunStepResult
{

View file

@ -0,0 +1,199 @@
// Copyright HyperTwist, Inc. All Rights Reserved.
#include "Misc/AutomationTest.h"
#include "Engine/GameInstance.h"
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
#include "UObject/UnrealType.h"
#if WITH_AUTOMATION_TESTS
namespace HyperTwistWhisperCppPhase6RETestInternal
{
FHyperTwistTrainingDeck MakeSpeechDeck()
{
FHyperTwistTrainingDeck Deck;
Deck.DeckId = TEXT("phase6r-e/whispercpp-speech");
Deck.Title = TEXT("Phase 6R-E Whisper Speech");
Deck.DeliveryModes = {
EHyperTwistTrainingDeliveryMode::CoachReviewed
};
FHyperTwistTrainingCase TrainingCase;
TrainingCase.CaseId = TEXT("phase6r-e-case");
TrainingCase.PuzzleId = TEXT("cube/3x3x3");
TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence;
TrainingCase.PromptLabel = TEXT("Phase 6R-E Coach Speech");
TrainingCase.AllowedDeliveryModes = {
EHyperTwistTrainingDeliveryMode::CoachReviewed
};
Deck.Cases = {TrainingCase};
return Deck;
}
void ForceMockSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem)
{
if (TrainingSubsystem == nullptr)
{
return;
}
if (FStrProperty* SpeechClientKindProperty = FindFProperty<FStrProperty>(
UHyperTwistTrainingSubsystem::StaticClass(),
TEXT("CompanionSpeechClientKind")
))
{
SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, TEXT("mock"));
}
}
UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId)
{
UGameInstance* GameInstance = NewObject<UGameInstance>(GetTransientPackage());
if (GameInstance == nullptr)
{
return nullptr;
}
UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject<UHyperTwistTrainingSubsystem>(GameInstance);
if (TrainingSubsystem == nullptr)
{
return nullptr;
}
ForceMockSpeechClient(TrainingSubsystem);
const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck(
MakeSpeechDeck(),
TEXT("phase6r-e-user"),
SessionId,
EHyperTwistTrainingDeliveryMode::CoachReviewed
);
return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr;
}
}
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
FHyperTwistWhisperCppPhase6RESessionConfigTest,
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.SessionConfig",
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
)
bool FHyperTwistWhisperCppPhase6RESessionConfigTest::RunTest(const FString& Parameters)
{
UHyperTwistTrainingSubsystem* TrainingSubsystem =
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-config-session"));
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
if (TrainingSubsystem == nullptr)
{
return false;
}
FString OpenError;
TestTrue(
TEXT("The companion speech session must open for a coach-reviewed run."),
TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)
);
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
TestEqual(
TEXT("The speech session must anchor on the existing listening contract."),
SessionState.SessionConfig.ListeningContractId,
TEXT("listening-threshold-lifecycle")
);
TestEqual(
TEXT("The speech session must preserve whisper.cpp mono PCM routing."),
SessionState.SessionConfig.AudioEncoding,
TEXT("pcm-f32le-16khz-mono")
);
TestEqual(
TEXT("The speech session must keep the bounded coach command grammar profile."),
SessionState.SessionConfig.GrammarProfileId,
TEXT("coach-command-grammar-v1")
);
TestTrue(TEXT("The speech session must keep VAD enabled."), SessionState.SessionConfig.bEnableVad);
TestTrue(
TEXT("The bounded route must preserve repeat-case as a command hint."),
SessionState.SessionConfig.CommandHints.Contains(TEXT("repeat case"))
);
TestTrue(TEXT("The mock speech health must report ready."), SessionState.ServiceHealth.bReady);
return true;
}
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
FHyperTwistWhisperCppPhase6RETranscriptResultTest,
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.TranscriptResult",
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
)
bool FHyperTwistWhisperCppPhase6RETranscriptResultTest::RunTest(const FString& Parameters)
{
UHyperTwistTrainingSubsystem* TrainingSubsystem =
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-transcript-session"));
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
if (TrainingSubsystem == nullptr)
{
return false;
}
FHyperTwistSpeechUtteranceEnvelope Utterance;
Utterance.AudioRef = TEXT("mock://coach/repeat-case");
Utterance.SpeechStartMs = 120;
Utterance.SpeechEndMs = 860;
Utterance.SilenceGapMs = 200;
Utterance.SampleCount = 11840;
const FHyperTwistSpeechTranscriptResult Result =
TrainingSubsystem->SubmitActiveCompanionSpeechUtterance(Utterance);
TestTrue(TEXT("The transcript result must be structurally valid."), Result.IsStructurallyValid());
TestEqual(TEXT("The bounded mock route must resolve the repeat-case transcript."), Result.TranscriptText, TEXT("repeat case"));
TestEqual(TEXT("The transcript result must keep one segment."), Result.Segments.Num(), 1);
if (Result.Segments.Num() > 0)
{
TestEqual(TEXT("The transcript segment must preserve the utterance start time."), Result.Segments[0].StartMs, 120);
TestEqual(TEXT("The transcript segment must preserve the utterance end time."), Result.Segments[0].EndMs, 860);
TestTrue(TEXT("The transcript segment must stay grammar-constrained."), Result.Segments[0].bGrammarConstrained);
}
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
TestEqual(TEXT("The active speech state must count one submitted utterance."), SessionState.SubmittedUtteranceCount, 1);
TestEqual(TEXT("The active speech state must count one final transcript."), SessionState.FinalTranscriptCount, 1);
TestEqual(TEXT("The active speech state must retain the last transcript text."), SessionState.LastTranscriptResult.TranscriptText, TEXT("repeat case"));
return true;
}
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
FHyperTwistWhisperCppPhase6RECloseSessionTest,
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.CloseSession",
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
)
bool FHyperTwistWhisperCppPhase6RECloseSessionTest::RunTest(const FString& Parameters)
{
UHyperTwistTrainingSubsystem* TrainingSubsystem =
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-close-session"));
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
if (TrainingSubsystem == nullptr)
{
return false;
}
FString OpenError;
TestTrue(TEXT("The speech session must open cleanly before close."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError));
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
FString CloseError;
TestTrue(TEXT("The speech session must close cleanly."), TrainingSubsystem->CloseActiveCompanionSpeechSession(CloseError));
TestTrue(TEXT("Closing the speech session must not report an error."), CloseError.IsEmpty());
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
TestFalse(TEXT("The speech session must no longer report as open."), SessionState.bSessionOpen);
TestTrue(TEXT("The mock speech health should remain available after close."), SessionState.ServiceHealth.bReady);
TestEqual(TEXT("The mock speech provider label must stay stable."), SessionState.ServiceHealth.ProviderLabel, TEXT("mock"));
return true;
}
#endif

View file

@ -1396,6 +1396,7 @@ Decision date:
- `2026-04-24`
- refreshed on `2026-05-13`
- implementation posture refreshed on `2026-05-21`
Current licensing judgment:
@ -1420,6 +1421,9 @@ Approved working posture:
- HyperTwist may use the codebase directly as the primary offline STT sidecar candidate under the `MIT` code posture
- keep it bounded behind a speech-input seam
- prefer command, dictation, and constrained coach-interaction use over broad “voice assistant platform” scope
- the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle
- keep model files, downloadable payloads, and future voice-asset review separate from the code-license judgment
- treat `SYSTRAN/faster-whisper` as the next complementary queue head for Python orchestration rather than widening `whisper.cpp` into the runtime core
### `SYSTRAN/faster-whisper`

View file

@ -0,0 +1,159 @@
# HyperTwist Phase 6R-E whisper.cpp speech transcript implementation packet
Created on `2026-05-21`
## Status
- first-party HyperTwist packet
- bounded permissive `Phase 6R-E` implementation slice
## Purpose
This packet lands the first narrower bounded slice from the retained `ggml-org/whisper.cpp`
speech-input row.
The landed slice is:
- first-party speech transcript session boundary above the existing companion listening lifecycle
It is not:
- a full `whisper.cpp` row transplant
- a live microphone capture packet
- a model checkout or payload-shipping packet
- a `SYSTRAN/faster-whisper` Python orchestration packet
- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet
- a broad voice-assistant platform packet
## Current authority basis
This implementation packet stands on:
- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md`
- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md`
- `docs/REPO_LICENSE_TRACKING.md`
- `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md`
The retained owner remains:
- `ggml-org/whisper.cpp`
The granted family remains bounded to:
- speech session configuration
- VAD-aware utterance-envelope shaping
- transcript-segment and transcript-result structure
- service-health and sidecar-boundary shaping
This packet lands only the first narrower family in that granted set.
## Landed scope
The current code now owns a retained speech transcript contract through:
- retained recognition contract types for:
- speech VAD policy
- speech session configuration
- speech utterance envelope
- speech transcript segment
- speech transcript result
- speech service health
- companion speech session state in:
- `FHyperTwistTrainingCompanionSpeechSessionState`
- sample session-config, transcript-result, and service-health outputs in:
- `UHyperTwistContractLibrary`
- direct-donor speech client seam with:
- `UHyperTwistMockSpeechClient`
- `UHyperTwistHttpSpeechClient`
- active companion open / submit / close transcript-session handling in:
- `UHyperTwistTrainingSubsystem`
- default sidecar configuration in:
- `DefaultGame.ini`
- focused automation coverage in:
- `HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp`
## Why this is still intentionally bounded
This packet lands the first donor-backed transcript-session seam, but it does not widen into the
neighboring retained families.
Still deferred:
- live microphone capture shell
- device-permission shell
- downloadable model or payload shipping
- Python-side orchestration, batching, or hotword service work
- TTS or voice-output ownership
- broad voice-assistant scope
## Validation
Build validation:
- `Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE`
Focused automation validation:
- `Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.E`
Regression automation validation:
- `Automation RunTests HyperTwist.Permissive.MagicTile.Phase6R.D`
- `Automation RunTests HyperTwist.Permissive.RubixCubeSolver.Phase6R.C`
- `Automation RunTests HyperTwist.Permissive.Qbr.Phase6R.B`
- `Automation RunTests HyperTwist.Permissive.Hyperspeedcube.Phase6R.A`
- `Automation RunTests HyperTwist.CleanRoom.CubeDesk`
Expected covered tests:
- `CloseSession`
- `SessionConfig`
- `TranscriptResult`
- `TopologyContract`
- `LoaderBoundary`
- `PackageChecklist`
- `CommitBuildsReconstructionSession`
- `RevisionLedger`
- `FinalizeBuildsClassicNet`
- `CalibrationSessionConfig`
- `PreviewObservation`
- `CommitObservation`
- `CatalogLookup`
- `SampleDefinition`
- `TrainingRunDefinition`
- existing `CubeDesk` clean-room regression suite
## Queue effect
This packet consumes the current `Phase 6R-E` implementation slice.
`ggml-org/whisper.cpp` remains only partially incorporated:
- landed now:
- speech transcript session boundary
- bounded VAD-aware session config
- utterance-envelope shape
- transcript segment and transcript result
- speech-service health and sidecar boundary
- still deferred:
- live microphone capture shell
- downloadable model or payload shipping
- Python orchestration / batching / hotword service layer
- TTS or voice-output ownership
- broad voice-assistant scope
The next clean move is not another immediate `whisper.cpp` widening by default.
The next queue head should be:
- a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` preparation/control pass
Keep the future sequencing guards visible:
- speech-input / voice sidecar set
- keep code-license judgments separate from model, voice, and payload-license review
- `SYSTRAN/faster-whisper`
- keep it complementary to `whisper.cpp` and bounded to Python transcription-service
orchestration rather than runtime-core ownership
- `rhasspy/piper` and `coqui-ai/TTS`
- keep future voice-asset review separate from the code-license judgment

View file

@ -0,0 +1,182 @@
# HyperTwist Phase 6R-E whisper.cpp speech transcript preparation packet
Created on `2026-05-21`
## Status
- historical same-day preparation authority
- bounded post-Phase-`6R-D` preparation slice
- the first bounded implementation slice now lands separately under:
- `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md`
## Purpose
This packet freezes the next widening order after the landed `Phase 6R-D` `roice3/MagicTile`
tiling-topology slice.
The open task was:
- define the first bounded source-backed widening packet for `ggml-org/whisper.cpp` as the
retained `Phase 6R-E` speech transcript session boundary
It is not:
- a full `whisper.cpp` row transplant
- a live microphone capture or device-permission packet
- a model checkout or downloadable payload-shipping packet
- a `SYSTRAN/faster-whisper` Python orchestration packet
- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet
- a broad “voice assistant platform” packet
## Current authority basis
This preparation packet stands on already-closed authority:
- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md`
- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md`
- `docs/REPO_LICENSE_TRACKING.md`
- `docs/arch/HYPERTWIST_PHASE6R_D_MAGICTILE_TILING_TOPOLOGY_IMPLEMENTATION_PACKET_2026-05-21.md`
The key accepted routing facts are:
- `ggml-org/whisper.cpp` is the retained offline STT donor immediately after the landed `6R-D`
slice
- earliest widening route is `Phase 6R / Packet 6R-E`
- the retained donor value there is:
- speech session configuration
- bounded VAD policy
- utterance-envelope shaping
- transcript-segment and transcript-result structure
- service-health / sidecar-boundary shaping
- ownership denied there is:
- do not let the row absorb microphone capture or permission shell ownership
- do not let the row absorb downloadable model, payload, or voice-asset review
- do not widen into Python orchestration that belongs with `SYSTRAN/faster-whisper`
- do not widen into TTS or broad voice-output ownership
- do not widen into a broad assistant platform
## Why this was the next packet
The earlier retained queue heads were already landed:
- `Phase 6R-A` `HactarCE/Hyperspeedcube`
- `Phase 6R-B` `kkoomen/qbr`
- `Phase 6R-C` `vivaansinghvi07/rubix-cube-solver`
- `Phase 6R-D` `roice3/MagicTile`
That advanced the live queue to:
- `Phase 6R-E` `ggml-org/whisper.cpp`
with the later adjacent retained rows ordered behind it:
- `Phase 6R-F` `SYSTRAN/faster-whisper`
- `rhasspy/piper`
- `coqui-ai/TTS`
## Required result
The source-backed control pass for this packet is now complete.
The first actual `6R-E` implementation packet should:
- use the retained `0R-A` and `2R-A` authority surfaces plus the inspected `whisper.cpp` source
basis
- define one bounded retained slice from `whisper.cpp`
- keep the slice inside the accepted speech-input seam
- widen only the first transcript-session family that can stand on its own without dragging in
capture-shell, model-shipping, or voice-output scope
- explicitly state which neighboring retained rows stay closed in that packet
## Source-backed retained basis
The queue-head decision is now source-backed rather than README-only.
Inspected retained donor basis:
- `README.md`
- `include/whisper.h`
- `examples/server/server.cpp`
Inspected first-party receiving basis:
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.cpp`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp`
## Narrowed 6R-E slice decision
The first widening slice was fixed as:
1. speech transcript session boundary above the existing listening lifecycle
not as the broader speech-input or voice platform.
That narrowed slice covers:
- first-party speech VAD policy
- first-party speech session configuration
- first-party utterance envelope
- first-party transcript segment and transcript result
- first-party speech-service health contract
- first-party companion speech session state
- contract-library sample session config and transcript outputs
- direct-donor speech client seam with mock and HTTP-sidecar implementations
- active companion open / submit / close transcript-session handling
- focused automation coverage
## Deferred neighboring families
The first `6R-E` implementation packet must keep these capability families closed:
- live microphone capture shell
- device-permission shell
- downloadable model or payload shipping
- Python orchestration, batching, or hotword service-layer work that belongs with
`SYSTRAN/faster-whisper`
- TTS or voice-output ownership
- broad “voice assistant platform” scope
Why this slice is first:
- `whisper.cpp` exposes the strongest narrow retained seam at bounded offline transcription with
VAD-aware session and transcript shape
- current first-party code already has a companion listening lifecycle but lacked an owned
transcript-session contract and sidecar boundary
- keeping model, payload, and voice decisions outside this packet preserves swapability and cleaner
legal review
## Out of scope for the first 6R-E packet
- live microphone capture
- downloadable model checkout or packaging
- TTS / voice output
- `faster-whisper` Python service orchestration
- `piper` or `coqui-ai/TTS`
- broad assistant or narration platform work
## Acceptance criteria
- the packet records why the live queue advanced from landed `6R-D` to `6R-E`
- the packet records that the first `6R-E` widening slice is the speech transcript session
boundary only
- the packet states exact out-of-scope families for the first `6R-E` pass
- the packet keeps the speech-lane legal sequencing guard visible:
- keep code-license judgments separate from model, voice, and payload-license review
- the packet leaves Python orchestration and voice-output rows deferred behind the landed slice
## Validation checklist
1. confirm the landed `Phase 6R-D` packet is the consumed prior queue head
2. confirm the retained `whisper.cpp` seam narrows cleanly to the transcript-session contract and
sidecar boundary
3. confirm the next implementation packet is framed as bounded source-backed widening rather than a
broad speech-platform transplant
That is the packet.

View file

@ -44,8 +44,14 @@ Status update on `2026-05-21`:
packet is now landed in current code
- `roice3/MagicTile` remains partially incorporated; transform-aware macro remapping, broader
non-Euclidean interaction, and host-shell ownership stay deferred
- the current next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar
preparation/control pass, starting with `ggml-org/whisper.cpp`
- the generic source-backed `Phase 6R-E` speech-input / voice sidecar control pass is now
consumed
- the bounded permissive `Phase 6R-E` `ggml-org/whisper.cpp` speech transcript session packet is
now landed in current code
- `ggml-org/whisper.cpp` remains partially incorporated; live microphone capture, model/payload
shipping, Python orchestration, and voice-output ownership stay deferred
- the current next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper`
preparation/control pass
- the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than
waiting on another first-party packet
@ -73,8 +79,8 @@ Donor-strength rule:
- route difficulty changes how retained value may enter the product, not whether it may win technically
- current shallow-eval set: `75` repos
- currently verified live/implemented in checked Unreal surfaces: `28`
- permissive live lanes: `17`
- currently verified live/implemented in checked Unreal surfaces: `29`
- permissive live lanes: `18`
- boundary-sensitive live lanes: `6`
- restrictive live lanes: `5`
- the restrictive landed lanes are:
@ -107,8 +113,8 @@ Current routing truth:
- active non-live implementation-board rows: `35`
- retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9`
The next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar
preparation/control pass, starting with `ggml-org/whisper.cpp`.
The next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper`
preparation/control pass.
Queue interpretation after that packet:
@ -153,8 +159,19 @@ Queue interpretation after that packet:
- transform-aware macro remapping
- broad non-Euclidean interaction shell
- WinForms/OpenTK host shell
- after the `Phase 6R-D` implementation packet, keep the next queue shape explicit:
- `ggml-org/whisper.cpp`
- `ggml-org/whisper.cpp` remains a partially landed row rather than a closed row:
- landed:
- speech transcript session boundary
- VAD-aware session config and utterance envelope
- transcript segment and transcript result
- sidecar health and speech client boundary
- still deferred:
- live microphone capture shell
- downloadable model or payload shipping
- Python orchestration / batching / hotword service layer
- TTS or voice-output ownership
- broad assistant-platform scope
- after the `Phase 6R-E` implementation packet, keep the next queue shape explicit:
- `SYSTRAN/faster-whisper`
- `rhasspy/piper`
- `coqui-ai/TTS`