Implement Phase 6R-E speech transcript session boundary
This commit is contained in:
parent
e0490fcf7d
commit
d0fc383801
15 changed files with 1905 additions and 8 deletions
|
|
@ -15,3 +15,12 @@ FinalizeSessionPath=/vision/session/finalize
|
|||
CloseSessionPath=/vision/session/close
|
||||
HealthPath=/vision/health
|
||||
RequestTimeoutSeconds=5.0
|
||||
|
||||
[/Script/UnrealHyperTwist.HyperTwistHttpSpeechClient]
|
||||
ProviderLabel=local-http-sidecar
|
||||
ServiceBaseUrl=http://127.0.0.1:8766
|
||||
OpenSessionPath=/speech/session/open
|
||||
TranscribeUtterancePath=/speech/utterance
|
||||
CloseSessionPath=/speech/session/close
|
||||
HealthPath=/speech/health
|
||||
RequestTimeoutSeconds=5.0
|
||||
|
|
|
|||
|
|
@ -612,6 +612,72 @@ FHyperTwistVisionServiceHealth UHyperTwistContractLibrary::MakeMockVisionService
|
|||
return Health;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechSessionConfig UHyperTwistContractLibrary::MakeSampleSpeechSessionConfig()
|
||||
{
|
||||
FHyperTwistSpeechSessionConfig Config;
|
||||
Config.SessionId = TEXT("speech_01");
|
||||
Config.ListeningContractId = TEXT("listening-threshold-lifecycle");
|
||||
Config.InputRouteId = TEXT("queue/listening");
|
||||
Config.AudioEncoding = TEXT("pcm-f32le-16khz-mono");
|
||||
Config.SampleRateHz = 16000;
|
||||
Config.ChannelCount = 1;
|
||||
Config.LanguageMode = TEXT("auto");
|
||||
Config.TaskKind = TEXT("transcribe");
|
||||
Config.GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
Config.bEnableVad = true;
|
||||
Config.bEnableWordTimestamps = false;
|
||||
Config.VadPolicy.Threshold = 0.5f;
|
||||
Config.VadPolicy.MinSpeechDurationMs = 250;
|
||||
Config.VadPolicy.MinSilenceDurationMs = 200;
|
||||
Config.VadPolicy.MaxSpeechDurationMs = 12000;
|
||||
Config.VadPolicy.SpeechPadMs = 100;
|
||||
Config.CommandHints = {
|
||||
TEXT("ready coach"),
|
||||
TEXT("repeat case"),
|
||||
TEXT("next case")
|
||||
};
|
||||
return Config;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechTranscriptResult UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult()
|
||||
{
|
||||
FHyperTwistSpeechTranscriptResult Result;
|
||||
Result.SessionId = TEXT("speech_01");
|
||||
Result.UtteranceId = TEXT("speech_01_utterance_001");
|
||||
Result.LanguageCode = TEXT("en");
|
||||
Result.TaskKind = TEXT("transcribe");
|
||||
Result.GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
Result.bIsFinal = true;
|
||||
Result.TranscriptText = TEXT("ready coach");
|
||||
|
||||
FHyperTwistSpeechTranscriptSegment Segment;
|
||||
Segment.SegmentOrdinal = 0;
|
||||
Segment.Text = Result.TranscriptText;
|
||||
Segment.StartMs = 120;
|
||||
Segment.EndMs = 840;
|
||||
Segment.Confidence = 0.94f;
|
||||
Segment.NoSpeechProbability = 0.04f;
|
||||
Segment.bGrammarConstrained = true;
|
||||
Result.Segments = {Segment};
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechServiceHealth()
|
||||
{
|
||||
FHyperTwistSpeechServiceHealth Health;
|
||||
Health.ProviderLabel = TEXT("mock");
|
||||
Health.ServiceVersion = TEXT("mock-speech/v1");
|
||||
Health.ServiceEndpoint = TEXT("in-process://mock");
|
||||
Health.Capabilities = {
|
||||
TEXT("transcribe"),
|
||||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints"),
|
||||
TEXT("sessioned-mock-client")
|
||||
};
|
||||
Health.bReady = true;
|
||||
return Health;
|
||||
}
|
||||
|
||||
FHyperTwistContentPack UHyperTwistContractLibrary::MakeSampleAlgTrainerContentPack()
|
||||
{
|
||||
return UHyperTwistTrainingCatalogLibrary::MakeAlgTrainerStarterContentPack();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,314 @@
|
|||
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
|
||||
|
||||
#include "Dom/JsonObject.h"
|
||||
#include "Http.h"
|
||||
#include "HttpManager.h"
|
||||
#include "HttpModule.h"
|
||||
#include "Interfaces/IHttpRequest.h"
|
||||
#include "Interfaces/IHttpResponse.h"
|
||||
#include "JsonObjectConverter.h"
|
||||
#include "Serialization/JsonSerializer.h"
|
||||
#include "Serialization/JsonWriter.h"
|
||||
|
||||
namespace HyperTwistHttpSpeechClientInternal
|
||||
{
|
||||
template <typename TStruct>
|
||||
bool SerializeStruct(const TStruct& Value, FString& OutJson)
|
||||
{
|
||||
return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0);
|
||||
}
|
||||
|
||||
template <typename TStruct>
|
||||
bool DeserializeStruct(const FString& Json, TStruct& OutValue)
|
||||
{
|
||||
return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0);
|
||||
}
|
||||
|
||||
FString MakeSessionIdJson(const FString& SessionId)
|
||||
{
|
||||
FString Json;
|
||||
const TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
|
||||
Root->SetStringField(TEXT("sessionId"), SessionId);
|
||||
const TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&Json);
|
||||
FJsonSerializer::Serialize(Root, Writer);
|
||||
return Json;
|
||||
}
|
||||
|
||||
FString BuildUrl(const FString& BaseUrl, const FString& Path)
|
||||
{
|
||||
if (BaseUrl.IsEmpty())
|
||||
{
|
||||
return FString();
|
||||
}
|
||||
|
||||
const FString TrimmedBase = BaseUrl.EndsWith(TEXT("/")) ? BaseUrl.LeftChop(1) : BaseUrl;
|
||||
if (Path.IsEmpty())
|
||||
{
|
||||
return TrimmedBase;
|
||||
}
|
||||
|
||||
const FString TrimmedPath = Path.StartsWith(TEXT("/")) ? Path.RightChop(1) : Path;
|
||||
return FString::Printf(TEXT("%s/%s"), *TrimmedBase, *TrimmedPath);
|
||||
}
|
||||
|
||||
struct FHttpJsonResponse
|
||||
{
|
||||
bool bCompleted = false;
|
||||
bool bSucceeded = false;
|
||||
int32 StatusCode = 0;
|
||||
FString ResponseBody;
|
||||
FString Error;
|
||||
};
|
||||
|
||||
bool ExecuteJsonRequest(
|
||||
const UHyperTwistHttpSpeechClient& Client,
|
||||
const FString& Verb,
|
||||
const FString& Url,
|
||||
const FString& RequestJson,
|
||||
FHttpJsonResponse& OutResponse)
|
||||
{
|
||||
OutResponse = FHttpJsonResponse();
|
||||
|
||||
if (Url.IsEmpty())
|
||||
{
|
||||
OutResponse.Error = TEXT("service-base-url-missing");
|
||||
return false;
|
||||
}
|
||||
|
||||
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
|
||||
Request->SetURL(Url);
|
||||
Request->SetVerb(Verb);
|
||||
Request->SetHeader(TEXT("Accept"), TEXT("application/json"));
|
||||
Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
|
||||
|
||||
if (!Client.AuthorizationToken.IsEmpty())
|
||||
{
|
||||
Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *Client.AuthorizationToken));
|
||||
}
|
||||
|
||||
if (!RequestJson.IsEmpty())
|
||||
{
|
||||
Request->SetContentAsString(RequestJson);
|
||||
}
|
||||
|
||||
Request->OnProcessRequestComplete().BindLambda(
|
||||
[&OutResponse](FHttpRequestPtr, FHttpResponsePtr Response, bool bWasSuccessful)
|
||||
{
|
||||
OutResponse.bCompleted = true;
|
||||
|
||||
if (Response.IsValid())
|
||||
{
|
||||
OutResponse.StatusCode = Response->GetResponseCode();
|
||||
OutResponse.ResponseBody = Response->GetContentAsString();
|
||||
}
|
||||
|
||||
if (!bWasSuccessful)
|
||||
{
|
||||
OutResponse.Error = TEXT("request-failed");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Response.IsValid())
|
||||
{
|
||||
OutResponse.Error = TEXT("response-missing");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!EHttpResponseCodes::IsOk(OutResponse.StatusCode))
|
||||
{
|
||||
OutResponse.Error = FString::Printf(TEXT("http-%d"), OutResponse.StatusCode);
|
||||
return;
|
||||
}
|
||||
|
||||
OutResponse.bSucceeded = true;
|
||||
});
|
||||
|
||||
if (!Request->ProcessRequest())
|
||||
{
|
||||
OutResponse.Error = TEXT("request-start-failed");
|
||||
return false;
|
||||
}
|
||||
|
||||
const double TimeoutSeconds = FMath::Max(static_cast<double>(Client.RequestTimeoutSeconds), 0.1);
|
||||
const double Deadline = FPlatformTime::Seconds() + TimeoutSeconds;
|
||||
while (!OutResponse.bCompleted && FPlatformTime::Seconds() < Deadline)
|
||||
{
|
||||
FHttpModule::Get().GetHttpManager().Tick(0.01f);
|
||||
FPlatformProcess::Sleep(0.01f);
|
||||
}
|
||||
|
||||
if (!OutResponse.bCompleted)
|
||||
{
|
||||
Request->CancelRequest();
|
||||
OutResponse.Error = TEXT("request-timeout");
|
||||
return false;
|
||||
}
|
||||
|
||||
return OutResponse.bSucceeded;
|
||||
}
|
||||
}
|
||||
|
||||
bool UHyperTwistHttpSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
FString RequestJson;
|
||||
if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Config, RequestJson))
|
||||
{
|
||||
OutError = TEXT("request-serialize-failed");
|
||||
LastTransportError = OutError;
|
||||
return false;
|
||||
}
|
||||
|
||||
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
|
||||
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
|
||||
*this,
|
||||
TEXT("POST"),
|
||||
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, OpenSessionPath),
|
||||
RequestJson,
|
||||
Response))
|
||||
{
|
||||
OutError = Response.Error;
|
||||
LastTransportError = OutError;
|
||||
return false;
|
||||
}
|
||||
|
||||
LastTransportError.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechTranscriptResult UHyperTwistHttpSpeechClient::TranscribeSpeechUtterance(
|
||||
const FHyperTwistSpeechUtteranceEnvelope& Utterance
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechTranscriptResult Result;
|
||||
Result.SessionId = Utterance.SessionId;
|
||||
Result.UtteranceId = Utterance.UtteranceId;
|
||||
|
||||
FString RequestJson;
|
||||
if (!HyperTwistHttpSpeechClientInternal::SerializeStruct(Utterance, RequestJson))
|
||||
{
|
||||
LastTransportError = TEXT("request-serialize-failed");
|
||||
Result.Warnings.Add(LastTransportError);
|
||||
return Result;
|
||||
}
|
||||
|
||||
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
|
||||
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
|
||||
*this,
|
||||
TEXT("POST"),
|
||||
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, TranscribeUtterancePath),
|
||||
RequestJson,
|
||||
Response))
|
||||
{
|
||||
LastTransportError = Response.Error;
|
||||
Result.Warnings.Add(LastTransportError);
|
||||
return Result;
|
||||
}
|
||||
|
||||
if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Result))
|
||||
{
|
||||
LastTransportError = TEXT("invalid-provider-response");
|
||||
Result = FHyperTwistSpeechTranscriptResult();
|
||||
Result.SessionId = Utterance.SessionId;
|
||||
Result.UtteranceId = Utterance.UtteranceId;
|
||||
Result.Warnings.Add(LastTransportError);
|
||||
return Result;
|
||||
}
|
||||
|
||||
if (Result.SessionId.IsEmpty())
|
||||
{
|
||||
Result.SessionId = Utterance.SessionId;
|
||||
}
|
||||
if (Result.UtteranceId.IsEmpty())
|
||||
{
|
||||
Result.UtteranceId = Utterance.UtteranceId;
|
||||
}
|
||||
|
||||
LastTransportError.Reset();
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistHttpSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
|
||||
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
|
||||
*this,
|
||||
TEXT("POST"),
|
||||
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, CloseSessionPath),
|
||||
HyperTwistHttpSpeechClientInternal::MakeSessionIdJson(SessionId),
|
||||
Response))
|
||||
{
|
||||
OutError = Response.Error;
|
||||
LastTransportError = OutError;
|
||||
return false;
|
||||
}
|
||||
|
||||
LastTransportError.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHealth() const
|
||||
{
|
||||
FHyperTwistSpeechServiceHealth Health;
|
||||
Health.ProviderLabel = ProviderLabel;
|
||||
Health.ServiceVersion = TEXT("provider-unavailable/v1");
|
||||
Health.ServiceEndpoint = ServiceBaseUrl;
|
||||
Health.bReady = false;
|
||||
Health.Capabilities = {
|
||||
TEXT("transport:http"),
|
||||
TEXT("provider-backed-speech"),
|
||||
TEXT("transcribe"),
|
||||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints")
|
||||
};
|
||||
|
||||
HyperTwistHttpSpeechClientInternal::FHttpJsonResponse Response;
|
||||
if (!HyperTwistHttpSpeechClientInternal::ExecuteJsonRequest(
|
||||
*this,
|
||||
TEXT("GET"),
|
||||
HyperTwistHttpSpeechClientInternal::BuildUrl(ServiceBaseUrl, HealthPath),
|
||||
FString(),
|
||||
Response))
|
||||
{
|
||||
LastTransportError = Response.Error;
|
||||
Health.LastError = LastTransportError;
|
||||
return Health;
|
||||
}
|
||||
|
||||
if (!HyperTwistHttpSpeechClientInternal::DeserializeStruct(Response.ResponseBody, Health))
|
||||
{
|
||||
LastTransportError = TEXT("invalid-provider-response");
|
||||
Health = FHyperTwistSpeechServiceHealth();
|
||||
Health.ProviderLabel = ProviderLabel;
|
||||
Health.ServiceVersion = TEXT("provider-unavailable/v1");
|
||||
Health.ServiceEndpoint = ServiceBaseUrl;
|
||||
Health.bReady = false;
|
||||
Health.Capabilities = {
|
||||
TEXT("transport:http"),
|
||||
TEXT("provider-backed-speech"),
|
||||
TEXT("transcribe"),
|
||||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints")
|
||||
};
|
||||
Health.LastError = LastTransportError;
|
||||
return Health;
|
||||
}
|
||||
|
||||
if (Health.ProviderLabel.IsEmpty())
|
||||
{
|
||||
Health.ProviderLabel = ProviderLabel;
|
||||
}
|
||||
if (Health.ServiceEndpoint.IsEmpty())
|
||||
{
|
||||
Health.ServiceEndpoint = ServiceBaseUrl;
|
||||
}
|
||||
|
||||
Health.Capabilities.AddUnique(TEXT("transport:http"));
|
||||
Health.Capabilities.AddUnique(TEXT("provider-backed-speech"));
|
||||
LastTransportError.Reset();
|
||||
Health.LastError.Reset();
|
||||
return Health;
|
||||
}
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
|
||||
|
||||
#include "HyperTwistBootstrap/HyperTwistContractLibrary.h"
|
||||
|
||||
namespace HyperTwistMockSpeechClientInternal
|
||||
{
|
||||
FString ResolveTranscriptText(
|
||||
const FHyperTwistSpeechSessionConfig* SessionConfig,
|
||||
const FHyperTwistSpeechUtteranceEnvelope& Utterance
|
||||
)
|
||||
{
|
||||
const FString NormalizedAudioRef = Utterance.AudioRef.ToLower();
|
||||
if (NormalizedAudioRef.Contains(TEXT("repeat-case")))
|
||||
{
|
||||
return TEXT("repeat case");
|
||||
}
|
||||
if (NormalizedAudioRef.Contains(TEXT("next-case")))
|
||||
{
|
||||
return TEXT("next case");
|
||||
}
|
||||
if (NormalizedAudioRef.Contains(TEXT("ready-coach")))
|
||||
{
|
||||
return TEXT("ready coach");
|
||||
}
|
||||
|
||||
if (Utterance.LocalCommandHints.Num() > 0 && !Utterance.LocalCommandHints[0].IsEmpty())
|
||||
{
|
||||
return Utterance.LocalCommandHints[0];
|
||||
}
|
||||
|
||||
if (SessionConfig != nullptr && SessionConfig->CommandHints.Num() > 0 && !SessionConfig->CommandHints[0].IsEmpty())
|
||||
{
|
||||
return SessionConfig->CommandHints[0];
|
||||
}
|
||||
|
||||
return TEXT("ready coach");
|
||||
}
|
||||
}
|
||||
|
||||
bool UHyperTwistMockSpeechClient::OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!Config.IsStructurallyValid())
|
||||
{
|
||||
OutError = TEXT("speech-session-config-invalid");
|
||||
return false;
|
||||
}
|
||||
|
||||
SessionConfigs.Add(Config.SessionId, Config);
|
||||
SessionUtteranceCounts.Add(Config.SessionId, 0);
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechTranscriptResult UHyperTwistMockSpeechClient::TranscribeSpeechUtterance(
|
||||
const FHyperTwistSpeechUtteranceEnvelope& Utterance
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechTranscriptResult Result = UHyperTwistContractLibrary::MakeMockSpeechTranscriptResult();
|
||||
Result.SessionId = Utterance.SessionId;
|
||||
Result.UtteranceId = Utterance.UtteranceId;
|
||||
|
||||
const FHyperTwistSpeechSessionConfig* SessionConfig = SessionConfigs.Find(Utterance.SessionId);
|
||||
if (SessionConfig == nullptr)
|
||||
{
|
||||
Result.Warnings.Add(TEXT("session-not-open"));
|
||||
return Result;
|
||||
}
|
||||
|
||||
const FString TranscriptText =
|
||||
HyperTwistMockSpeechClientInternal::ResolveTranscriptText(SessionConfig, Utterance);
|
||||
|
||||
SessionUtteranceCounts.Add(Utterance.SessionId, SessionUtteranceCounts.FindRef(Utterance.SessionId) + 1);
|
||||
|
||||
Result.LanguageCode = SessionConfig->LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase)
|
||||
? TEXT("en")
|
||||
: SessionConfig->LanguageMode;
|
||||
Result.TaskKind = SessionConfig->TaskKind;
|
||||
Result.GrammarProfileId = SessionConfig->GrammarProfileId;
|
||||
Result.bIsFinal = true;
|
||||
Result.TranscriptText = TranscriptText;
|
||||
Result.Segments.Reset();
|
||||
|
||||
FHyperTwistSpeechTranscriptSegment Segment;
|
||||
Segment.SegmentOrdinal = 0;
|
||||
Segment.Text = TranscriptText;
|
||||
Segment.StartMs = FMath::Max(0, Utterance.SpeechStartMs);
|
||||
Segment.EndMs = FMath::Max(Segment.StartMs, Utterance.SpeechEndMs);
|
||||
Segment.Confidence = 0.93f;
|
||||
Segment.NoSpeechProbability = 0.05f;
|
||||
Segment.bGrammarConstrained = !SessionConfig->GrammarProfileId.IsEmpty();
|
||||
Result.Segments.Add(Segment);
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistMockSpeechClient::CloseSpeechSession(const FString& SessionId, FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!SessionConfigs.Contains(SessionId))
|
||||
{
|
||||
OutError = TEXT("session-not-open");
|
||||
return false;
|
||||
}
|
||||
|
||||
SessionConfigs.Remove(SessionId);
|
||||
SessionUtteranceCounts.Remove(SessionId);
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHealth() const
|
||||
{
|
||||
FHyperTwistSpeechServiceHealth Health = UHyperTwistContractLibrary::MakeMockSpeechServiceHealth();
|
||||
Health.ProviderLabel = TEXT("mock");
|
||||
Health.ServiceEndpoint = TEXT("in-process://mock");
|
||||
Health.Capabilities.AddUnique(TEXT("offline-transcribe"));
|
||||
return Health;
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
|
||||
|
||||
#include "HyperTwistRecognition/HyperTwistVisionClient.h"
|
||||
#include "HyperTwistRecognition/HyperTwistSpeechClient.h"
|
||||
#include "HyperTwistRecognition/HyperTwistRecognitionReplayLibrary.h"
|
||||
#include "HyperTwistReplay/HyperTwistReplayLibrary.h"
|
||||
#include "HyperTwistReplay/HyperTwistReplayReviewLibrary.h"
|
||||
|
|
@ -11,6 +12,7 @@
|
|||
#include "HyperTwistTraining/HyperTwistTrainingPersistenceLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingPublicationLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingRepositoryLibrary.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h"
|
||||
#include "JsonObjectConverter.h"
|
||||
|
||||
namespace HyperTwistTrainingSubsystemInternal
|
||||
|
|
@ -1855,17 +1857,35 @@ namespace HyperTwistTrainingSubsystemInternal
|
|||
: nullptr;
|
||||
}
|
||||
|
||||
IHyperTwistSpeechClient* ResolveSpeechClient(UObject* Candidate)
|
||||
{
|
||||
return Candidate != nullptr && Candidate->GetClass()->ImplementsInterface(UHyperTwistSpeechClient::StaticClass())
|
||||
? Cast<IHyperTwistSpeechClient>(Candidate)
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
FString BuildRecognitionSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
|
||||
{
|
||||
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
|
||||
return FString::Printf(TEXT("%s_recognition_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
|
||||
}
|
||||
|
||||
FString BuildSpeechSessionId(const FHyperTwistTrainingSession& Session, const int32 SessionOrdinal)
|
||||
{
|
||||
const int32 ResolvedOrdinal = FMath::Max(1, SessionOrdinal);
|
||||
return FString::Printf(TEXT("%s_speech_%02d"), *Session.TrainingSessionId, ResolvedOrdinal);
|
||||
}
|
||||
|
||||
FString BuildRecognitionFrameId(const FString& SessionId, const int32 FrameOrdinal)
|
||||
{
|
||||
return FString::Printf(TEXT("%s_frame_%03d"), *SessionId, FMath::Max(1, FrameOrdinal));
|
||||
}
|
||||
|
||||
FString BuildSpeechUtteranceId(const FString& SessionId, const int32 UtteranceOrdinal)
|
||||
{
|
||||
return FString::Printf(TEXT("%s_utterance_%03d"), *SessionId, FMath::Max(1, UtteranceOrdinal));
|
||||
}
|
||||
|
||||
bool IsQbrClassicCubeRoute(const FHyperTwistVisionSessionConfig& SessionConfig)
|
||||
{
|
||||
return SessionConfig.ExpectedPuzzleFamily == EHyperTwistPuzzleFamily::ClassicCube
|
||||
|
|
@ -3609,6 +3629,11 @@ FHyperTwistTrainingRecognitionSessionState UHyperTwistTrainingSubsystem::GetActi
|
|||
return ActiveRecognitionSessionState;
|
||||
}
|
||||
|
||||
FHyperTwistTrainingCompanionSpeechSessionState UHyperTwistTrainingSubsystem::GetActiveCompanionSpeechSessionState() const
|
||||
{
|
||||
return ActiveCompanionSpeechSessionState;
|
||||
}
|
||||
|
||||
TArray<FHyperTwistCoachSignal> UHyperTwistTrainingSubsystem::GetActiveCoachSignals() const
|
||||
{
|
||||
return ActiveCoachSignals;
|
||||
|
|
@ -5487,7 +5512,13 @@ FHyperTwistTrainingRunState UHyperTwistTrainingSubsystem::StartTrainingRunFromDe
|
|||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
if (ActiveCompanionSpeechSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveCompanionSpeechSession(CloseError);
|
||||
}
|
||||
ResetRecognitionSessionState();
|
||||
ResetCompanionSpeechSessionState();
|
||||
ActiveReviewPlanState = FHyperTwistTrainingReviewPlanState();
|
||||
ActiveReviewFlowStatus = FHyperTwistTrainingReviewFlowStatus();
|
||||
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
|
||||
|
|
@ -6360,6 +6391,16 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
|
|||
bAbortSession ? TEXT("training-run-aborted") : TEXT("training-run-completed")
|
||||
);
|
||||
}
|
||||
if (ActiveRecognitionSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
if (ActiveCompanionSpeechSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveCompanionSpeechSession(CloseError);
|
||||
}
|
||||
|
||||
ResetActiveLiveTimerState();
|
||||
|
||||
|
|
@ -6418,6 +6459,7 @@ FHyperTwistTrainingSession UHyperTwistTrainingSubsystem::CompleteActiveRun(bool
|
|||
}
|
||||
}
|
||||
ResetRecognitionSessionState();
|
||||
ResetCompanionSpeechSessionState();
|
||||
RefreshRepositoryViews();
|
||||
return ActiveRunState.Session;
|
||||
}
|
||||
|
|
@ -6429,6 +6471,11 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun()
|
|||
FString CloseError;
|
||||
CloseActiveRecognitionSession(CloseError);
|
||||
}
|
||||
if (ActiveCompanionSpeechSessionState.bSessionOpen)
|
||||
{
|
||||
FString CloseError;
|
||||
CloseActiveCompanionSpeechSession(CloseError);
|
||||
}
|
||||
if (HasActiveSmartDeviceSession())
|
||||
{
|
||||
StopActiveSmartDeviceSession(TEXT("training-run-cleared"));
|
||||
|
|
@ -6466,6 +6513,7 @@ void UHyperTwistTrainingSubsystem::ClearActiveRun()
|
|||
ActiveCoachActionPlan = FHyperTwistTrainingCoachActionPlan();
|
||||
bHasActiveRun = false;
|
||||
ResetRecognitionSessionState();
|
||||
ResetCompanionSpeechSessionState();
|
||||
RefreshRepositoryViews();
|
||||
}
|
||||
|
||||
|
|
@ -7043,6 +7091,226 @@ bool UHyperTwistTrainingSubsystem::CloseActiveRecognitionSession(FString& OutErr
|
|||
return true;
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
OutError = TEXT("no-active-training-run");
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
|
||||
ResolveCompanionSpeechClientObject()
|
||||
);
|
||||
if (SpeechClient == nullptr)
|
||||
{
|
||||
OutError = TEXT("speech-client-unavailable");
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FHyperTwistSpeechSessionConfig SessionConfig = BuildActiveCompanionSpeechSessionConfig();
|
||||
if (SessionConfig.SessionId.IsEmpty())
|
||||
{
|
||||
OutError = TEXT("speech-session-id-unavailable");
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ActiveCompanionSpeechSessionState.bSessionOpen
|
||||
&& ActiveCompanionSpeechSessionState.ActiveSessionId == SessionConfig.SessionId)
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
|
||||
ActiveCompanionSpeechSessionState.LastError.Reset();
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ActiveCompanionSpeechSessionState.bSessionOpen
|
||||
&& ActiveCompanionSpeechSessionState.ActiveSessionId != SessionConfig.SessionId)
|
||||
{
|
||||
FString CloseError;
|
||||
SpeechClient->CloseSpeechSession(ActiveCompanionSpeechSessionState.ActiveSessionId, CloseError);
|
||||
}
|
||||
|
||||
if (!SpeechClient->OpenSpeechSession(SessionConfig, OutError))
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FString OpenedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveCompanionSpeechSessionState.bSessionOpen = true;
|
||||
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient =
|
||||
ActiveCompanionSpeechClientObject != nullptr
|
||||
&& ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>();
|
||||
ActiveCompanionSpeechSessionState.ActiveSessionId = SessionConfig.SessionId;
|
||||
ActiveCompanionSpeechSessionState.OpenedAtUtc = OpenedAtUtc;
|
||||
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = OpenedAtUtc;
|
||||
ActiveCompanionSpeechSessionState.LastError.Reset();
|
||||
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
|
||||
ActiveCompanionSpeechSessionState.bHasTranscriptResult = false;
|
||||
ActiveCompanionSpeechSessionState.LastTranscriptResult = FHyperTwistSpeechTranscriptResult();
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveCompanionSpeechUtterance(
|
||||
const FHyperTwistSpeechUtteranceEnvelope& Utterance
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechTranscriptResult Result;
|
||||
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
Result.Warnings.Add(TEXT("no-active-training-run"));
|
||||
return Result;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
if (!OpenActiveCompanionSpeechSession(OpenError))
|
||||
{
|
||||
Result.SessionId = !Utterance.SessionId.IsEmpty()
|
||||
? Utterance.SessionId
|
||||
: BuildActiveCompanionSpeechSessionConfig().SessionId;
|
||||
Result.UtteranceId = Utterance.UtteranceId;
|
||||
Result.Warnings.Add(OpenError.IsEmpty() ? TEXT("speech-session-open-failed") : OpenError);
|
||||
return Result;
|
||||
}
|
||||
|
||||
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
|
||||
ResolveCompanionSpeechClientObject()
|
||||
);
|
||||
if (SpeechClient == nullptr)
|
||||
{
|
||||
Result.Warnings.Add(TEXT("speech-client-unavailable"));
|
||||
ActiveCompanionSpeechSessionState.LastError = Result.Warnings[0];
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError;
|
||||
return Result;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechUtteranceEnvelope NormalizedUtterance = Utterance;
|
||||
NormalizedUtterance.SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId;
|
||||
if (NormalizedUtterance.UtteranceId.IsEmpty())
|
||||
{
|
||||
NormalizedUtterance.UtteranceId = HyperTwistTrainingSubsystemInternal::BuildSpeechUtteranceId(
|
||||
NormalizedUtterance.SessionId,
|
||||
ActiveCompanionSpeechSessionState.SubmittedUtteranceCount + 1
|
||||
);
|
||||
}
|
||||
if (NormalizedUtterance.CapturedAtUtc.IsEmpty())
|
||||
{
|
||||
NormalizedUtterance.CapturedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
}
|
||||
if (NormalizedUtterance.SampleRateHz <= 0)
|
||||
{
|
||||
NormalizedUtterance.SampleRateHz = ActiveCompanionSpeechSessionState.SessionConfig.SampleRateHz;
|
||||
}
|
||||
if (NormalizedUtterance.ChannelCount <= 0)
|
||||
{
|
||||
NormalizedUtterance.ChannelCount = ActiveCompanionSpeechSessionState.SessionConfig.ChannelCount;
|
||||
}
|
||||
if (NormalizedUtterance.LocalCommandHints.Num() <= 0)
|
||||
{
|
||||
NormalizedUtterance.LocalCommandHints = ActiveCompanionSpeechSessionState.SessionConfig.CommandHints;
|
||||
}
|
||||
NormalizedUtterance.SilenceGapMs = FMath::Max(0, NormalizedUtterance.SilenceGapMs);
|
||||
NormalizedUtterance.SpeechStartMs = FMath::Max(0, NormalizedUtterance.SpeechStartMs);
|
||||
NormalizedUtterance.SpeechEndMs = FMath::Max(
|
||||
NormalizedUtterance.SpeechStartMs,
|
||||
NormalizedUtterance.SpeechEndMs
|
||||
);
|
||||
|
||||
Result = SpeechClient->TranscribeSpeechUtterance(NormalizedUtterance);
|
||||
if (Result.SessionId.IsEmpty())
|
||||
{
|
||||
Result.SessionId = NormalizedUtterance.SessionId;
|
||||
}
|
||||
if (Result.UtteranceId.IsEmpty())
|
||||
{
|
||||
Result.UtteranceId = NormalizedUtterance.UtteranceId;
|
||||
}
|
||||
if (Result.TaskKind.IsEmpty())
|
||||
{
|
||||
Result.TaskKind = ActiveCompanionSpeechSessionState.SessionConfig.TaskKind;
|
||||
}
|
||||
if (Result.GrammarProfileId.IsEmpty())
|
||||
{
|
||||
Result.GrammarProfileId = ActiveCompanionSpeechSessionState.SessionConfig.GrammarProfileId;
|
||||
}
|
||||
if (Result.TranscriptText.IsEmpty() && Result.Segments.Num() > 0)
|
||||
{
|
||||
TArray<FString> SegmentTexts;
|
||||
for (const FHyperTwistSpeechTranscriptSegment& Segment : Result.Segments)
|
||||
{
|
||||
if (!Segment.Text.IsEmpty())
|
||||
{
|
||||
SegmentTexts.Add(Segment.Text);
|
||||
}
|
||||
}
|
||||
Result.TranscriptText = FString::Join(SegmentTexts, TEXT(" "));
|
||||
}
|
||||
|
||||
++ActiveCompanionSpeechSessionState.SubmittedUtteranceCount;
|
||||
if (Result.bIsFinal && Result.IsStructurallyValid())
|
||||
{
|
||||
++ActiveCompanionSpeechSessionState.FinalTranscriptCount;
|
||||
}
|
||||
ActiveCompanionSpeechSessionState.bHasTranscriptResult = true;
|
||||
ActiveCompanionSpeechSessionState.LastTranscriptResult = Result;
|
||||
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveCompanionSpeechSessionState.LastError =
|
||||
Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString();
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError;
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::CloseActiveCompanionSpeechSession(FString& OutError)
|
||||
{
|
||||
OutError.Reset();
|
||||
|
||||
if (!ActiveCompanionSpeechSessionState.bSessionOpen || ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty())
|
||||
{
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
|
||||
ResolveCompanionSpeechClientObject()
|
||||
);
|
||||
if (SpeechClient == nullptr)
|
||||
{
|
||||
OutError = TEXT("speech-client-unavailable");
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
const FString SessionId = ActiveCompanionSpeechSessionState.ActiveSessionId;
|
||||
const bool bClosed = SpeechClient->CloseSpeechSession(SessionId, OutError);
|
||||
if (!bClosed && OutError != TEXT("session-not-open"))
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.LastError = OutError;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return false;
|
||||
}
|
||||
|
||||
ActiveCompanionSpeechSessionState.bSessionOpen = false;
|
||||
ActiveCompanionSpeechSessionState.ActiveSessionId.Reset();
|
||||
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveCompanionSpeechSessionState.LastError.Reset();
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::ClearActiveMethodDrillRun()
|
||||
{
|
||||
ActiveMethodDrillRunState = FHyperTwistTrainingMethodDrillRunState();
|
||||
|
|
@ -7410,6 +7678,74 @@ FHyperTwistVisionSessionConfig UHyperTwistTrainingSubsystem::BuildActiveRecognit
|
|||
return SessionConfig;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechSessionConfig UHyperTwistTrainingSubsystem::BuildActiveCompanionSpeechSessionConfig() const
|
||||
{
|
||||
FHyperTwistSpeechSessionConfig SessionConfig = ActiveCompanionSpeechSessionState.SessionConfig;
|
||||
if (!HasActiveRun())
|
||||
{
|
||||
return SessionConfig;
|
||||
}
|
||||
|
||||
const FHyperTwistTrainingCompanionReferenceBundle CompanionBundle =
|
||||
UHyperTwistTrainingRuntimeLibrary::GetBundledEmbodiedCompanionReferenceBundle();
|
||||
|
||||
SessionConfig.SessionId = !ActiveCompanionSpeechSessionState.ActiveSessionId.IsEmpty()
|
||||
? ActiveCompanionSpeechSessionState.ActiveSessionId
|
||||
: HyperTwistTrainingSubsystemInternal::BuildSpeechSessionId(ActiveRunState.Session, 1);
|
||||
SessionConfig.ListeningContractId = !CompanionBundle.PrimaryListeningContractId.IsEmpty()
|
||||
? CompanionBundle.PrimaryListeningContractId
|
||||
: TEXT("listening-threshold-lifecycle");
|
||||
if (SessionConfig.InputRouteId.IsEmpty())
|
||||
{
|
||||
SessionConfig.InputRouteId = TEXT("queue/listening");
|
||||
}
|
||||
if (SessionConfig.AudioEncoding.IsEmpty())
|
||||
{
|
||||
SessionConfig.AudioEncoding = TEXT("pcm-f32le-16khz-mono");
|
||||
}
|
||||
if (SessionConfig.SampleRateHz <= 0)
|
||||
{
|
||||
SessionConfig.SampleRateHz = 16000;
|
||||
}
|
||||
if (SessionConfig.ChannelCount <= 0)
|
||||
{
|
||||
SessionConfig.ChannelCount = 1;
|
||||
}
|
||||
if (SessionConfig.LanguageMode.IsEmpty())
|
||||
{
|
||||
SessionConfig.LanguageMode = TEXT("auto");
|
||||
}
|
||||
if (SessionConfig.TaskKind.IsEmpty())
|
||||
{
|
||||
SessionConfig.TaskKind = TEXT("transcribe");
|
||||
}
|
||||
if (SessionConfig.GrammarProfileId.IsEmpty())
|
||||
{
|
||||
SessionConfig.GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
}
|
||||
SessionConfig.bEnableVad = true;
|
||||
SessionConfig.bEnableWordTimestamps = false;
|
||||
if (!SessionConfig.VadPolicy.IsStructurallyValid())
|
||||
{
|
||||
SessionConfig.VadPolicy.Threshold = 0.5f;
|
||||
SessionConfig.VadPolicy.MinSpeechDurationMs = 250;
|
||||
SessionConfig.VadPolicy.MinSilenceDurationMs = 200;
|
||||
SessionConfig.VadPolicy.MaxSpeechDurationMs = 12000;
|
||||
SessionConfig.VadPolicy.SpeechPadMs = 100;
|
||||
}
|
||||
if (SessionConfig.CommandHints.Num() <= 0)
|
||||
{
|
||||
SessionConfig.CommandHints = {
|
||||
TEXT("ready coach"),
|
||||
TEXT("repeat case"),
|
||||
TEXT("next case"),
|
||||
TEXT("pause coaching")
|
||||
};
|
||||
}
|
||||
|
||||
return SessionConfig;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
|
||||
{
|
||||
IHyperTwistVisionClient* VisionClient = HyperTwistTrainingSubsystemInternal::ResolveVisionClient(
|
||||
|
|
@ -7435,6 +7771,58 @@ void UHyperTwistTrainingSubsystem::RefreshRecognitionServiceHealth()
|
|||
ActiveRecognitionSessionState.ServiceHealth = VisionClient->GetVisionServiceHealth();
|
||||
}
|
||||
|
||||
UObject* UHyperTwistTrainingSubsystem::ResolveCompanionSpeechClientObject()
|
||||
{
|
||||
LoadConfig();
|
||||
|
||||
const bool bUseMockClient = CompanionSpeechClientKind.Equals(TEXT("mock"), ESearchCase::IgnoreCase);
|
||||
const bool bNeedsNewClient = ActiveCompanionSpeechClientObject == nullptr
|
||||
|| (bUseMockClient && !ActiveCompanionSpeechClientObject->IsA<UHyperTwistMockSpeechClient>())
|
||||
|| (!bUseMockClient && !ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>());
|
||||
if (bNeedsNewClient)
|
||||
{
|
||||
if (bUseMockClient)
|
||||
{
|
||||
ActiveCompanionSpeechClientObject = NewObject<UHyperTwistMockSpeechClient>(this);
|
||||
}
|
||||
else
|
||||
{
|
||||
UHyperTwistHttpSpeechClient* HttpSpeechClient = NewObject<UHyperTwistHttpSpeechClient>(this);
|
||||
HttpSpeechClient->LoadConfig();
|
||||
ActiveCompanionSpeechClientObject = HttpSpeechClient;
|
||||
}
|
||||
}
|
||||
|
||||
return ActiveCompanionSpeechClientObject;
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::RefreshCompanionSpeechServiceHealth()
|
||||
{
|
||||
IHyperTwistSpeechClient* SpeechClient = HyperTwistTrainingSubsystemInternal::ResolveSpeechClient(
|
||||
ResolveCompanionSpeechClientObject()
|
||||
);
|
||||
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient =
|
||||
ActiveCompanionSpeechClientObject != nullptr
|
||||
&& ActiveCompanionSpeechClientObject->IsA<UHyperTwistHttpSpeechClient>();
|
||||
if (SpeechClient == nullptr)
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth = FHyperTwistSpeechServiceHealth();
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.ProviderLabel =
|
||||
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient
|
||||
? TEXT("local-http-sidecar")
|
||||
: TEXT("mock");
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.ServiceEndpoint =
|
||||
ActiveCompanionSpeechSessionState.bUsingProviderBackedClient
|
||||
? TEXT("http://127.0.0.1:8766")
|
||||
: TEXT("in-process://mock");
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.bReady = false;
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = TEXT("speech-client-unavailable");
|
||||
return;
|
||||
}
|
||||
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth = SpeechClient->GetSpeechServiceHealth();
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent(
|
||||
const EHyperTwistReplayEventType EventType,
|
||||
const FHyperTwistReplayRecognitionPayload& Payload,
|
||||
|
|
@ -7492,6 +7880,12 @@ void UHyperTwistTrainingSubsystem::ResetRecognitionSessionState()
|
|||
RefreshRecognitionServiceHealth();
|
||||
}
|
||||
|
||||
void UHyperTwistTrainingSubsystem::ResetCompanionSpeechSessionState()
|
||||
{
|
||||
ActiveCompanionSpeechSessionState = FHyperTwistTrainingCompanionSpeechSessionState();
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
}
|
||||
|
||||
bool UHyperTwistTrainingSubsystem::TryResolveRepositoryViewContext(
|
||||
FString& OutUserId,
|
||||
FString& OutDeckId,
|
||||
|
|
|
|||
|
|
@ -57,6 +57,15 @@ public:
|
|||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
|
||||
static FHyperTwistVisionServiceHealth MakeMockVisionServiceHealth();
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
|
||||
static FHyperTwistSpeechSessionConfig MakeSampleSpeechSessionConfig();
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
|
||||
static FHyperTwistSpeechTranscriptResult MakeMockSpeechTranscriptResult();
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
|
||||
static FHyperTwistSpeechServiceHealth MakeMockSpeechServiceHealth();
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training")
|
||||
static FHyperTwistContentPack MakeSampleAlgTrainerContentPack();
|
||||
|
||||
|
|
|
|||
|
|
@ -572,3 +572,263 @@ struct FHyperTwistVisionServiceHealth
|
|||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bReady = true;
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechVadPolicy
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float Threshold = 0.5f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 MinSpeechDurationMs = 250;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 MinSilenceDurationMs = 200;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 MaxSpeechDurationMs = 12000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SpeechPadMs = 100;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return Threshold > 0.0f
|
||||
&& MinSpeechDurationMs >= 0
|
||||
&& MinSilenceDurationMs >= 0
|
||||
&& MaxSpeechDurationMs >= MinSpeechDurationMs
|
||||
&& SpeechPadMs >= 0;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechSessionConfig
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString SessionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ListeningContractId = TEXT("listening-threshold-lifecycle");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString InputRouteId = TEXT("queue/listening");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString AudioEncoding = TEXT("pcm-f32le-16khz-mono");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SampleRateHz = 16000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 ChannelCount = 1;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LanguageMode = TEXT("auto");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString TaskKind = TEXT("transcribe");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bEnableVad = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bEnableWordTimestamps = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechVadPolicy VadPolicy;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FString> CommandHints;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return !SessionId.IsEmpty()
|
||||
&& !ListeningContractId.IsEmpty()
|
||||
&& !InputRouteId.IsEmpty()
|
||||
&& !AudioEncoding.IsEmpty()
|
||||
&& SampleRateHz > 0
|
||||
&& ChannelCount > 0
|
||||
&& !TaskKind.IsEmpty()
|
||||
&& VadPolicy.IsStructurallyValid();
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechUtteranceEnvelope
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString SessionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString UtteranceId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString CapturedAtUtc;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString AudioRef;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SourceTimestampMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SampleCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SampleRateHz = 16000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 ChannelCount = 1;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SpeechStartMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SpeechEndMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SilenceGapMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float EnergyThreshold = 0.0f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FString> LocalCommandHints;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSpeechCommitReady = true;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return !SessionId.IsEmpty()
|
||||
&& !UtteranceId.IsEmpty()
|
||||
&& SampleRateHz > 0
|
||||
&& ChannelCount > 0
|
||||
&& SpeechStartMs >= 0
|
||||
&& SpeechEndMs >= SpeechStartMs
|
||||
&& SilenceGapMs >= 0;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechTranscriptSegment
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SegmentOrdinal = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString Text;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 StartMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 EndMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float Confidence = 0.0f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float NoSpeechProbability = 0.0f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bGrammarConstrained = false;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return SegmentOrdinal >= 0
|
||||
&& !Text.IsEmpty()
|
||||
&& StartMs >= 0
|
||||
&& EndMs >= StartMs
|
||||
&& Confidence >= 0.0f;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechTranscriptResult
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString SessionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString UtteranceId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LanguageCode = TEXT("en");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString TaskKind = TEXT("transcribe");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bIsFinal = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString TranscriptText;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FHyperTwistSpeechTranscriptSegment> Segments;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FString> Warnings;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
if (SessionId.IsEmpty()
|
||||
|| UtteranceId.IsEmpty()
|
||||
|| TaskKind.IsEmpty()
|
||||
|| (TranscriptText.IsEmpty() && Segments.Num() <= 0))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const FHyperTwistSpeechTranscriptSegment& Segment : Segments)
|
||||
{
|
||||
if (!Segment.IsStructurallyValid())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechServiceHealth
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ProviderLabel = TEXT("mock");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ServiceVersion = TEXT("mock-speech/v1");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ServiceEndpoint;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FString> Capabilities;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastError;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bReady = true;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,96 @@
|
|||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "UObject/Interface.h"
|
||||
#include "UObject/Object.h"
|
||||
#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h"
|
||||
#include "HyperTwistSpeechClient.generated.h"
|
||||
|
||||
UINTERFACE(BlueprintType)
|
||||
class UNREALHYPERTWIST_API UHyperTwistSpeechClient : public UInterface
|
||||
{
|
||||
GENERATED_BODY()
|
||||
};
|
||||
|
||||
class UNREALHYPERTWIST_API IHyperTwistSpeechClient
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) = 0;
|
||||
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) = 0;
|
||||
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) = 0;
|
||||
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const = 0;
|
||||
};
|
||||
|
||||
UCLASS(BlueprintType)
|
||||
class UNREALHYPERTWIST_API UHyperTwistMockSpeechClient : public UObject, public IHyperTwistSpeechClient
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override;
|
||||
|
||||
private:
|
||||
UPROPERTY()
|
||||
TMap<FString, FHyperTwistSpeechSessionConfig> SessionConfigs;
|
||||
|
||||
UPROPERTY()
|
||||
TMap<FString, int32> SessionUtteranceCounts;
|
||||
};
|
||||
|
||||
UCLASS(BlueprintType, Config = Game, DefaultConfig)
|
||||
class UNREALHYPERTWIST_API UHyperTwistHttpSpeechClient : public UObject, public IHyperTwistSpeechClient
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString ProviderLabel = TEXT("local-http-sidecar");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString ServiceBaseUrl = TEXT("http://127.0.0.1:8766");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString OpenSessionPath = TEXT("/speech/session/open");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString TranscribeUtterancePath = TEXT("/speech/utterance");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString CloseSessionPath = TEXT("/speech/session/close");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString HealthPath = TEXT("/speech/health");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
float RequestTimeoutSeconds = 5.0f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
|
||||
FString AuthorizationToken;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool OpenSpeechSession(const FHyperTwistSpeechSessionConfig& Config, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistSpeechTranscriptResult TranscribeSpeechUtterance(const FHyperTwistSpeechUtteranceEnvelope& Utterance) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual bool CloseSpeechSession(const FString& SessionId, FString& OutError) override;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
|
||||
virtual FHyperTwistSpeechServiceHealth GetSpeechServiceHealth() const override;
|
||||
|
||||
private:
|
||||
mutable FString LastTransportError;
|
||||
};
|
||||
|
|
@ -176,6 +176,9 @@ public:
|
|||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Recognition")
|
||||
FHyperTwistTrainingRecognitionSessionState GetActiveRecognitionSessionState() const;
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Speech")
|
||||
FHyperTwistTrainingCompanionSpeechSessionState GetActiveCompanionSpeechSessionState() const;
|
||||
|
||||
UFUNCTION(BlueprintPure, Category = "HyperTwist|Training|Coach")
|
||||
TArray<FHyperTwistCoachSignal> GetActiveCoachSignals() const;
|
||||
|
||||
|
|
@ -685,6 +688,17 @@ public:
|
|||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Recognition")
|
||||
bool CloseActiveRecognitionSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
|
||||
bool OpenActiveCompanionSpeechSession(FString& OutError);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
|
||||
FHyperTwistSpeechTranscriptResult SubmitActiveCompanionSpeechUtterance(
|
||||
const FHyperTwistSpeechUtteranceEnvelope& Utterance
|
||||
);
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Training|Speech")
|
||||
bool CloseActiveCompanionSpeechSession(FString& OutError);
|
||||
|
||||
void SetActiveAlgJsSequence(const FHyperTwistAlgorithmSequence& Sequence);
|
||||
bool TryGetActiveAlgJsSequence(FHyperTwistAlgorithmSequence& OutSequence) const;
|
||||
|
||||
|
|
@ -716,8 +730,11 @@ private:
|
|||
const FString& NowUtc
|
||||
) const;
|
||||
FHyperTwistVisionSessionConfig BuildActiveRecognitionSessionConfig() const;
|
||||
FHyperTwistSpeechSessionConfig BuildActiveCompanionSpeechSessionConfig() const;
|
||||
UObject* ResolveRecognitionVisionClientObject();
|
||||
UObject* ResolveCompanionSpeechClientObject();
|
||||
void RefreshRecognitionServiceHealth();
|
||||
void RefreshCompanionSpeechServiceHealth();
|
||||
void AppendRecognitionReplayEvent(
|
||||
EHyperTwistReplayEventType EventType,
|
||||
const FHyperTwistReplayRecognitionPayload& Payload,
|
||||
|
|
@ -725,6 +742,7 @@ private:
|
|||
);
|
||||
void SynchronizeRecognitionReplayMutation();
|
||||
void ResetRecognitionSessionState();
|
||||
void ResetCompanionSpeechSessionState();
|
||||
TArray<FHyperTwistTrainingSessionTemplate> BuildTrainingSessionTemplatesForUser(
|
||||
const FString& UserId,
|
||||
const FString& ReferenceUtc
|
||||
|
|
@ -820,6 +838,9 @@ private:
|
|||
UPROPERTY()
|
||||
FHyperTwistTrainingRecognitionSessionState ActiveRecognitionSessionState;
|
||||
|
||||
UPROPERTY()
|
||||
FHyperTwistTrainingCompanionSpeechSessionState ActiveCompanionSpeechSessionState;
|
||||
|
||||
UPROPERTY()
|
||||
TArray<FHyperTwistCoachSignal> ActiveCoachSignals;
|
||||
|
||||
|
|
@ -898,6 +919,12 @@ private:
|
|||
UPROPERTY()
|
||||
TObjectPtr<UObject> ActiveRecognitionVisionClientObject;
|
||||
|
||||
UPROPERTY(Config)
|
||||
FString CompanionSpeechClientKind = TEXT("http-sidecar");
|
||||
|
||||
UPROPERTY()
|
||||
TObjectPtr<UObject> ActiveCompanionSpeechClientObject;
|
||||
|
||||
UPROPERTY()
|
||||
bool bHasActiveRun = false;
|
||||
|
||||
|
|
|
|||
|
|
@ -1890,6 +1890,48 @@ struct FHyperTwistTrainingRecognitionSessionState
|
|||
FHyperTwistVisionReconstructionSession ActiveReconstructionSession;
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistTrainingCompanionSpeechSessionState
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSessionOpen = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bUsingProviderBackedClient = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bHasTranscriptResult = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ActiveSessionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString OpenedAtUtc;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastUpdatedAtUtc;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastError;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SubmittedUtteranceCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 FinalTranscriptCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechSessionConfig SessionConfig;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechServiceHealth ServiceHealth;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechTranscriptResult LastTranscriptResult;
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistTrainingRunStepResult
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,199 @@
|
|||
// Copyright HyperTwist, Inc. All Rights Reserved.
|
||||
|
||||
#include "Misc/AutomationTest.h"
|
||||
|
||||
#include "Engine/GameInstance.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
|
||||
#include "UObject/UnrealType.h"
|
||||
|
||||
#if WITH_AUTOMATION_TESTS
|
||||
|
||||
namespace HyperTwistWhisperCppPhase6RETestInternal
|
||||
{
|
||||
FHyperTwistTrainingDeck MakeSpeechDeck()
|
||||
{
|
||||
FHyperTwistTrainingDeck Deck;
|
||||
Deck.DeckId = TEXT("phase6r-e/whispercpp-speech");
|
||||
Deck.Title = TEXT("Phase 6R-E Whisper Speech");
|
||||
Deck.DeliveryModes = {
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
};
|
||||
|
||||
FHyperTwistTrainingCase TrainingCase;
|
||||
TrainingCase.CaseId = TEXT("phase6r-e-case");
|
||||
TrainingCase.PuzzleId = TEXT("cube/3x3x3");
|
||||
TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence;
|
||||
TrainingCase.PromptLabel = TEXT("Phase 6R-E Coach Speech");
|
||||
TrainingCase.AllowedDeliveryModes = {
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
};
|
||||
Deck.Cases = {TrainingCase};
|
||||
return Deck;
|
||||
}
|
||||
|
||||
void ForceMockSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem)
|
||||
{
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (FStrProperty* SpeechClientKindProperty = FindFProperty<FStrProperty>(
|
||||
UHyperTwistTrainingSubsystem::StaticClass(),
|
||||
TEXT("CompanionSpeechClientKind")
|
||||
))
|
||||
{
|
||||
SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, TEXT("mock"));
|
||||
}
|
||||
}
|
||||
|
||||
UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId)
|
||||
{
|
||||
UGameInstance* GameInstance = NewObject<UGameInstance>(GetTransientPackage());
|
||||
if (GameInstance == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject<UHyperTwistTrainingSubsystem>(GameInstance);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ForceMockSpeechClient(TrainingSubsystem);
|
||||
const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck(
|
||||
MakeSpeechDeck(),
|
||||
TEXT("phase6r-e-user"),
|
||||
SessionId,
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
);
|
||||
return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RESessionConfigTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.SessionConfig",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RESessionConfigTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-config-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
TestTrue(
|
||||
TEXT("The companion speech session must open for a coach-reviewed run."),
|
||||
TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)
|
||||
);
|
||||
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
TestEqual(
|
||||
TEXT("The speech session must anchor on the existing listening contract."),
|
||||
SessionState.SessionConfig.ListeningContractId,
|
||||
TEXT("listening-threshold-lifecycle")
|
||||
);
|
||||
TestEqual(
|
||||
TEXT("The speech session must preserve whisper.cpp mono PCM routing."),
|
||||
SessionState.SessionConfig.AudioEncoding,
|
||||
TEXT("pcm-f32le-16khz-mono")
|
||||
);
|
||||
TestEqual(
|
||||
TEXT("The speech session must keep the bounded coach command grammar profile."),
|
||||
SessionState.SessionConfig.GrammarProfileId,
|
||||
TEXT("coach-command-grammar-v1")
|
||||
);
|
||||
TestTrue(TEXT("The speech session must keep VAD enabled."), SessionState.SessionConfig.bEnableVad);
|
||||
TestTrue(
|
||||
TEXT("The bounded route must preserve repeat-case as a command hint."),
|
||||
SessionState.SessionConfig.CommandHints.Contains(TEXT("repeat case"))
|
||||
);
|
||||
TestTrue(TEXT("The mock speech health must report ready."), SessionState.ServiceHealth.bReady);
|
||||
return true;
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RETranscriptResultTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.TranscriptResult",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RETranscriptResultTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-transcript-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechUtteranceEnvelope Utterance;
|
||||
Utterance.AudioRef = TEXT("mock://coach/repeat-case");
|
||||
Utterance.SpeechStartMs = 120;
|
||||
Utterance.SpeechEndMs = 860;
|
||||
Utterance.SilenceGapMs = 200;
|
||||
Utterance.SampleCount = 11840;
|
||||
const FHyperTwistSpeechTranscriptResult Result =
|
||||
TrainingSubsystem->SubmitActiveCompanionSpeechUtterance(Utterance);
|
||||
|
||||
TestTrue(TEXT("The transcript result must be structurally valid."), Result.IsStructurallyValid());
|
||||
TestEqual(TEXT("The bounded mock route must resolve the repeat-case transcript."), Result.TranscriptText, TEXT("repeat case"));
|
||||
TestEqual(TEXT("The transcript result must keep one segment."), Result.Segments.Num(), 1);
|
||||
if (Result.Segments.Num() > 0)
|
||||
{
|
||||
TestEqual(TEXT("The transcript segment must preserve the utterance start time."), Result.Segments[0].StartMs, 120);
|
||||
TestEqual(TEXT("The transcript segment must preserve the utterance end time."), Result.Segments[0].EndMs, 860);
|
||||
TestTrue(TEXT("The transcript segment must stay grammar-constrained."), Result.Segments[0].bGrammarConstrained);
|
||||
}
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
TestEqual(TEXT("The active speech state must count one submitted utterance."), SessionState.SubmittedUtteranceCount, 1);
|
||||
TestEqual(TEXT("The active speech state must count one final transcript."), SessionState.FinalTranscriptCount, 1);
|
||||
TestEqual(TEXT("The active speech state must retain the last transcript text."), SessionState.LastTranscriptResult.TranscriptText, TEXT("repeat case"));
|
||||
return true;
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RECloseSessionTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.E.CloseSession",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RECloseSessionTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RETestInternal::MakeSpeechSubsystem(TEXT("phase6r-e-close-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-E."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
TestTrue(TEXT("The speech session must open cleanly before close."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError));
|
||||
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
|
||||
|
||||
FString CloseError;
|
||||
TestTrue(TEXT("The speech session must close cleanly."), TrainingSubsystem->CloseActiveCompanionSpeechSession(CloseError));
|
||||
TestTrue(TEXT("Closing the speech session must not report an error."), CloseError.IsEmpty());
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
TestFalse(TEXT("The speech session must no longer report as open."), SessionState.bSessionOpen);
|
||||
TestTrue(TEXT("The mock speech health should remain available after close."), SessionState.ServiceHealth.bReady);
|
||||
TestEqual(TEXT("The mock speech provider label must stay stable."), SessionState.ServiceHealth.ProviderLabel, TEXT("mock"));
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -1396,6 +1396,7 @@ Decision date:
|
|||
|
||||
- `2026-04-24`
|
||||
- refreshed on `2026-05-13`
|
||||
- implementation posture refreshed on `2026-05-21`
|
||||
|
||||
Current licensing judgment:
|
||||
|
||||
|
|
@ -1420,6 +1421,9 @@ Approved working posture:
|
|||
- HyperTwist may use the codebase directly as the primary offline STT sidecar candidate under the `MIT` code posture
|
||||
- keep it bounded behind a speech-input seam
|
||||
- prefer command, dictation, and constrained coach-interaction use over broad “voice assistant platform” scope
|
||||
- the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle
|
||||
- keep model files, downloadable payloads, and future voice-asset review separate from the code-license judgment
|
||||
- treat `SYSTRAN/faster-whisper` as the next complementary queue head for Python orchestration rather than widening `whisper.cpp` into the runtime core
|
||||
|
||||
### `SYSTRAN/faster-whisper`
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
# HyperTwist Phase 6R-E whisper.cpp speech transcript implementation packet
|
||||
|
||||
Created on `2026-05-21`
|
||||
|
||||
## Status
|
||||
|
||||
- first-party HyperTwist packet
|
||||
- bounded permissive `Phase 6R-E` implementation slice
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet lands the first narrower bounded slice from the retained `ggml-org/whisper.cpp`
|
||||
speech-input row.
|
||||
|
||||
The landed slice is:
|
||||
|
||||
- first-party speech transcript session boundary above the existing companion listening lifecycle
|
||||
|
||||
It is not:
|
||||
|
||||
- a full `whisper.cpp` row transplant
|
||||
- a live microphone capture packet
|
||||
- a model checkout or payload-shipping packet
|
||||
- a `SYSTRAN/faster-whisper` Python orchestration packet
|
||||
- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet
|
||||
- a broad voice-assistant platform packet
|
||||
|
||||
## Current authority basis
|
||||
|
||||
This implementation packet stands on:
|
||||
|
||||
- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md`
|
||||
- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md`
|
||||
- `docs/REPO_LICENSE_TRACKING.md`
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_PREPARATION_PACKET_2026-05-21.md`
|
||||
|
||||
The retained owner remains:
|
||||
|
||||
- `ggml-org/whisper.cpp`
|
||||
|
||||
The granted family remains bounded to:
|
||||
|
||||
- speech session configuration
|
||||
- VAD-aware utterance-envelope shaping
|
||||
- transcript-segment and transcript-result structure
|
||||
- service-health and sidecar-boundary shaping
|
||||
|
||||
This packet lands only the first narrower family in that granted set.
|
||||
|
||||
## Landed scope
|
||||
|
||||
The current code now owns a retained speech transcript contract through:
|
||||
|
||||
- retained recognition contract types for:
|
||||
- speech VAD policy
|
||||
- speech session configuration
|
||||
- speech utterance envelope
|
||||
- speech transcript segment
|
||||
- speech transcript result
|
||||
- speech service health
|
||||
- companion speech session state in:
|
||||
- `FHyperTwistTrainingCompanionSpeechSessionState`
|
||||
- sample session-config, transcript-result, and service-health outputs in:
|
||||
- `UHyperTwistContractLibrary`
|
||||
- direct-donor speech client seam with:
|
||||
- `UHyperTwistMockSpeechClient`
|
||||
- `UHyperTwistHttpSpeechClient`
|
||||
- active companion open / submit / close transcript-session handling in:
|
||||
- `UHyperTwistTrainingSubsystem`
|
||||
- default sidecar configuration in:
|
||||
- `DefaultGame.ini`
|
||||
- focused automation coverage in:
|
||||
- `HyperTwistWhisperCppPhase6RESpeechTranscriptContractTest.cpp`
|
||||
|
||||
## Why this is still intentionally bounded
|
||||
|
||||
This packet lands the first donor-backed transcript-session seam, but it does not widen into the
|
||||
neighboring retained families.
|
||||
|
||||
Still deferred:
|
||||
|
||||
- live microphone capture shell
|
||||
- device-permission shell
|
||||
- downloadable model or payload shipping
|
||||
- Python-side orchestration, batching, or hotword service work
|
||||
- TTS or voice-output ownership
|
||||
- broad voice-assistant scope
|
||||
|
||||
## Validation
|
||||
|
||||
Build validation:
|
||||
|
||||
- `Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE`
|
||||
|
||||
Focused automation validation:
|
||||
|
||||
- `Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.E`
|
||||
|
||||
Regression automation validation:
|
||||
|
||||
- `Automation RunTests HyperTwist.Permissive.MagicTile.Phase6R.D`
|
||||
- `Automation RunTests HyperTwist.Permissive.RubixCubeSolver.Phase6R.C`
|
||||
- `Automation RunTests HyperTwist.Permissive.Qbr.Phase6R.B`
|
||||
- `Automation RunTests HyperTwist.Permissive.Hyperspeedcube.Phase6R.A`
|
||||
- `Automation RunTests HyperTwist.CleanRoom.CubeDesk`
|
||||
|
||||
Expected covered tests:
|
||||
|
||||
- `CloseSession`
|
||||
- `SessionConfig`
|
||||
- `TranscriptResult`
|
||||
- `TopologyContract`
|
||||
- `LoaderBoundary`
|
||||
- `PackageChecklist`
|
||||
- `CommitBuildsReconstructionSession`
|
||||
- `RevisionLedger`
|
||||
- `FinalizeBuildsClassicNet`
|
||||
- `CalibrationSessionConfig`
|
||||
- `PreviewObservation`
|
||||
- `CommitObservation`
|
||||
- `CatalogLookup`
|
||||
- `SampleDefinition`
|
||||
- `TrainingRunDefinition`
|
||||
- existing `CubeDesk` clean-room regression suite
|
||||
|
||||
## Queue effect
|
||||
|
||||
This packet consumes the current `Phase 6R-E` implementation slice.
|
||||
|
||||
`ggml-org/whisper.cpp` remains only partially incorporated:
|
||||
|
||||
- landed now:
|
||||
- speech transcript session boundary
|
||||
- bounded VAD-aware session config
|
||||
- utterance-envelope shape
|
||||
- transcript segment and transcript result
|
||||
- speech-service health and sidecar boundary
|
||||
- still deferred:
|
||||
- live microphone capture shell
|
||||
- downloadable model or payload shipping
|
||||
- Python orchestration / batching / hotword service layer
|
||||
- TTS or voice-output ownership
|
||||
- broad voice-assistant scope
|
||||
|
||||
The next clean move is not another immediate `whisper.cpp` widening by default.
|
||||
|
||||
The next queue head should be:
|
||||
|
||||
- a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` preparation/control pass
|
||||
|
||||
Keep the future sequencing guards visible:
|
||||
|
||||
- speech-input / voice sidecar set
|
||||
- keep code-license judgments separate from model, voice, and payload-license review
|
||||
- `SYSTRAN/faster-whisper`
|
||||
- keep it complementary to `whisper.cpp` and bounded to Python transcription-service
|
||||
orchestration rather than runtime-core ownership
|
||||
- `rhasspy/piper` and `coqui-ai/TTS`
|
||||
- keep future voice-asset review separate from the code-license judgment
|
||||
|
|
@ -0,0 +1,182 @@
|
|||
# HyperTwist Phase 6R-E whisper.cpp speech transcript preparation packet
|
||||
|
||||
Created on `2026-05-21`
|
||||
|
||||
## Status
|
||||
|
||||
- historical same-day preparation authority
|
||||
- bounded post-Phase-`6R-D` preparation slice
|
||||
- the first bounded implementation slice now lands separately under:
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_E_WHISPER_CPP_SPEECH_TRANSCRIPT_IMPLEMENTATION_PACKET_2026-05-21.md`
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet freezes the next widening order after the landed `Phase 6R-D` `roice3/MagicTile`
|
||||
tiling-topology slice.
|
||||
|
||||
The open task was:
|
||||
|
||||
- define the first bounded source-backed widening packet for `ggml-org/whisper.cpp` as the
|
||||
retained `Phase 6R-E` speech transcript session boundary
|
||||
|
||||
It is not:
|
||||
|
||||
- a full `whisper.cpp` row transplant
|
||||
- a live microphone capture or device-permission packet
|
||||
- a model checkout or downloadable payload-shipping packet
|
||||
- a `SYSTRAN/faster-whisper` Python orchestration packet
|
||||
- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet
|
||||
- a broad “voice assistant platform” packet
|
||||
|
||||
## Current authority basis
|
||||
|
||||
This preparation packet stands on already-closed authority:
|
||||
|
||||
- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md`
|
||||
- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md`
|
||||
- `docs/REPO_LICENSE_TRACKING.md`
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_D_MAGICTILE_TILING_TOPOLOGY_IMPLEMENTATION_PACKET_2026-05-21.md`
|
||||
|
||||
The key accepted routing facts are:
|
||||
|
||||
- `ggml-org/whisper.cpp` is the retained offline STT donor immediately after the landed `6R-D`
|
||||
slice
|
||||
- earliest widening route is `Phase 6R / Packet 6R-E`
|
||||
- the retained donor value there is:
|
||||
- speech session configuration
|
||||
- bounded VAD policy
|
||||
- utterance-envelope shaping
|
||||
- transcript-segment and transcript-result structure
|
||||
- service-health / sidecar-boundary shaping
|
||||
- ownership denied there is:
|
||||
- do not let the row absorb microphone capture or permission shell ownership
|
||||
- do not let the row absorb downloadable model, payload, or voice-asset review
|
||||
- do not widen into Python orchestration that belongs with `SYSTRAN/faster-whisper`
|
||||
- do not widen into TTS or broad voice-output ownership
|
||||
- do not widen into a broad assistant platform
|
||||
|
||||
## Why this was the next packet
|
||||
|
||||
The earlier retained queue heads were already landed:
|
||||
|
||||
- `Phase 6R-A` `HactarCE/Hyperspeedcube`
|
||||
- `Phase 6R-B` `kkoomen/qbr`
|
||||
- `Phase 6R-C` `vivaansinghvi07/rubix-cube-solver`
|
||||
- `Phase 6R-D` `roice3/MagicTile`
|
||||
|
||||
That advanced the live queue to:
|
||||
|
||||
- `Phase 6R-E` `ggml-org/whisper.cpp`
|
||||
|
||||
with the later adjacent retained rows ordered behind it:
|
||||
|
||||
- `Phase 6R-F` `SYSTRAN/faster-whisper`
|
||||
- `rhasspy/piper`
|
||||
- `coqui-ai/TTS`
|
||||
|
||||
## Required result
|
||||
|
||||
The source-backed control pass for this packet is now complete.
|
||||
|
||||
The first actual `6R-E` implementation packet should:
|
||||
|
||||
- use the retained `0R-A` and `2R-A` authority surfaces plus the inspected `whisper.cpp` source
|
||||
basis
|
||||
- define one bounded retained slice from `whisper.cpp`
|
||||
- keep the slice inside the accepted speech-input seam
|
||||
- widen only the first transcript-session family that can stand on its own without dragging in
|
||||
capture-shell, model-shipping, or voice-output scope
|
||||
- explicitly state which neighboring retained rows stay closed in that packet
|
||||
|
||||
## Source-backed retained basis
|
||||
|
||||
The queue-head decision is now source-backed rather than README-only.
|
||||
|
||||
Inspected retained donor basis:
|
||||
|
||||
- `README.md`
|
||||
- `include/whisper.h`
|
||||
- `examples/server/server.cpp`
|
||||
|
||||
Inspected first-party receiving basis:
|
||||
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingCompanionLibrary.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp`
|
||||
|
||||
## Narrowed 6R-E slice decision
|
||||
|
||||
The first widening slice was fixed as:
|
||||
|
||||
1. speech transcript session boundary above the existing listening lifecycle
|
||||
|
||||
not as the broader speech-input or voice platform.
|
||||
|
||||
That narrowed slice covers:
|
||||
|
||||
- first-party speech VAD policy
|
||||
- first-party speech session configuration
|
||||
- first-party utterance envelope
|
||||
- first-party transcript segment and transcript result
|
||||
- first-party speech-service health contract
|
||||
- first-party companion speech session state
|
||||
- contract-library sample session config and transcript outputs
|
||||
- direct-donor speech client seam with mock and HTTP-sidecar implementations
|
||||
- active companion open / submit / close transcript-session handling
|
||||
- focused automation coverage
|
||||
|
||||
## Deferred neighboring families
|
||||
|
||||
The first `6R-E` implementation packet must keep these capability families closed:
|
||||
|
||||
- live microphone capture shell
|
||||
- device-permission shell
|
||||
- downloadable model or payload shipping
|
||||
- Python orchestration, batching, or hotword service-layer work that belongs with
|
||||
`SYSTRAN/faster-whisper`
|
||||
- TTS or voice-output ownership
|
||||
- broad “voice assistant platform” scope
|
||||
|
||||
Why this slice is first:
|
||||
|
||||
- `whisper.cpp` exposes the strongest narrow retained seam at bounded offline transcription with
|
||||
VAD-aware session and transcript shape
|
||||
- current first-party code already has a companion listening lifecycle but lacked an owned
|
||||
transcript-session contract and sidecar boundary
|
||||
- keeping model, payload, and voice decisions outside this packet preserves swapability and cleaner
|
||||
legal review
|
||||
|
||||
## Out of scope for the first 6R-E packet
|
||||
|
||||
- live microphone capture
|
||||
- downloadable model checkout or packaging
|
||||
- TTS / voice output
|
||||
- `faster-whisper` Python service orchestration
|
||||
- `piper` or `coqui-ai/TTS`
|
||||
- broad assistant or narration platform work
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- the packet records why the live queue advanced from landed `6R-D` to `6R-E`
|
||||
- the packet records that the first `6R-E` widening slice is the speech transcript session
|
||||
boundary only
|
||||
- the packet states exact out-of-scope families for the first `6R-E` pass
|
||||
- the packet keeps the speech-lane legal sequencing guard visible:
|
||||
- keep code-license judgments separate from model, voice, and payload-license review
|
||||
- the packet leaves Python orchestration and voice-output rows deferred behind the landed slice
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. confirm the landed `Phase 6R-D` packet is the consumed prior queue head
|
||||
2. confirm the retained `whisper.cpp` seam narrows cleanly to the transcript-session contract and
|
||||
sidecar boundary
|
||||
3. confirm the next implementation packet is framed as bounded source-backed widening rather than a
|
||||
broad speech-platform transplant
|
||||
|
||||
That is the packet.
|
||||
|
|
@ -44,8 +44,14 @@ Status update on `2026-05-21`:
|
|||
packet is now landed in current code
|
||||
- `roice3/MagicTile` remains partially incorporated; transform-aware macro remapping, broader
|
||||
non-Euclidean interaction, and host-shell ownership stay deferred
|
||||
- the current next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar
|
||||
preparation/control pass, starting with `ggml-org/whisper.cpp`
|
||||
- the generic source-backed `Phase 6R-E` speech-input / voice sidecar control pass is now
|
||||
consumed
|
||||
- the bounded permissive `Phase 6R-E` `ggml-org/whisper.cpp` speech transcript session packet is
|
||||
now landed in current code
|
||||
- `ggml-org/whisper.cpp` remains partially incorporated; live microphone capture, model/payload
|
||||
shipping, Python orchestration, and voice-output ownership stay deferred
|
||||
- the current next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper`
|
||||
preparation/control pass
|
||||
- the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than
|
||||
waiting on another first-party packet
|
||||
|
||||
|
|
@ -73,8 +79,8 @@ Donor-strength rule:
|
|||
- route difficulty changes how retained value may enter the product, not whether it may win technically
|
||||
|
||||
- current shallow-eval set: `75` repos
|
||||
- currently verified live/implemented in checked Unreal surfaces: `28`
|
||||
- permissive live lanes: `17`
|
||||
- currently verified live/implemented in checked Unreal surfaces: `29`
|
||||
- permissive live lanes: `18`
|
||||
- boundary-sensitive live lanes: `6`
|
||||
- restrictive live lanes: `5`
|
||||
- the restrictive landed lanes are:
|
||||
|
|
@ -107,8 +113,8 @@ Current routing truth:
|
|||
- active non-live implementation-board rows: `35`
|
||||
- retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9`
|
||||
|
||||
The next bounded move is a source-backed `Phase 6R-E` speech-input / voice sidecar
|
||||
preparation/control pass, starting with `ggml-org/whisper.cpp`.
|
||||
The next bounded move is a source-backed `Phase 6R-F` `SYSTRAN/faster-whisper`
|
||||
preparation/control pass.
|
||||
|
||||
Queue interpretation after that packet:
|
||||
|
||||
|
|
@ -153,8 +159,19 @@ Queue interpretation after that packet:
|
|||
- transform-aware macro remapping
|
||||
- broad non-Euclidean interaction shell
|
||||
- WinForms/OpenTK host shell
|
||||
- after the `Phase 6R-D` implementation packet, keep the next queue shape explicit:
|
||||
- `ggml-org/whisper.cpp`
|
||||
- `ggml-org/whisper.cpp` remains a partially landed row rather than a closed row:
|
||||
- landed:
|
||||
- speech transcript session boundary
|
||||
- VAD-aware session config and utterance envelope
|
||||
- transcript segment and transcript result
|
||||
- sidecar health and speech client boundary
|
||||
- still deferred:
|
||||
- live microphone capture shell
|
||||
- downloadable model or payload shipping
|
||||
- Python orchestration / batching / hotword service layer
|
||||
- TTS or voice-output ownership
|
||||
- broad assistant-platform scope
|
||||
- after the `Phase 6R-E` implementation packet, keep the next queue shape explicit:
|
||||
- `SYSTRAN/faster-whisper`
|
||||
- `rhasspy/piper`
|
||||
- `coqui-ai/TTS`
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue