From c25a2d7331256cb51b782efda62e03c31348cbaa Mon Sep 17 00:00:00 2001 From: axiomlogicnexus Date: Fri, 22 May 2026 02:13:11 +0200 Subject: [PATCH] Implement Phase 6R-G local narration sidecar contract --- UnrealHyperTwist/Config/DefaultGame.ini | 8 + .../HyperTwistContractLibrary.cpp | 185 ++++++++ .../HyperTwistHttpVoiceClient.cpp | 440 ++++++++++++++++++ .../HyperTwistMockVoiceClient.cpp | 68 +++ .../HyperTwistContractLibrary.h | 12 + .../HyperTwistRecognitionTypes.h | 251 ++++++++++ .../HyperTwistVoiceClient.h | 85 ++++ ...iperPhase6RGLocalNarrationContractTest.cpp | 134 ++++++ docs/REPO_LICENSE_TRACKING.md | 24 +- ...IDECAR_IMPLEMENTATION_PACKET_2026-05-22.md | 134 ++++++ ...N_SIDECAR_PREPARATION_PACKET_2026-05-22.md | 76 +++ ...PERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md | 6 +- .../HyperTwist/FEATURE_REGISTRY.md | 3 +- .../HyperTwist/ROADMAP.md | 36 +- 14 files changed, 1443 insertions(+), 19 deletions(-) create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVoiceClient.cpp create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVoiceClient.cpp create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVoiceClient.h create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistPiperPhase6RGLocalNarrationContractTest.cpp create mode 100644 docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_IMPLEMENTATION_PACKET_2026-05-22.md create mode 100644 docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_PREPARATION_PACKET_2026-05-22.md diff --git a/UnrealHyperTwist/Config/DefaultGame.ini b/UnrealHyperTwist/Config/DefaultGame.ini index 1cc847f..1ddef48 100644 --- a/UnrealHyperTwist/Config/DefaultGame.ini +++ b/UnrealHyperTwist/Config/DefaultGame.ini @@ -24,3 +24,11 @@ TranscribeUtterancePath=/speech/utterance CloseSessionPath=/speech/session/close HealthPath=/speech/health RequestTimeoutSeconds=5.0 + +[/Script/UnrealHyperTwist.HyperTwistHttpVoiceClient] +ProviderLabel=local-http-sidecar +ServiceBaseUrl=http://127.0.0.1:8766 +ProfilesPath=/voice/profiles +SynthesizePath=/voice/narrate +HealthPath=/voice/health +RequestTimeoutSeconds=5.0 diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp index 2ff0293..b2499c9 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp @@ -24,6 +24,78 @@ namespace HyperTwistContractLibraryInternal return FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0); } + void AppendLe16(TArray& Buffer, const uint16 Value) + { + Buffer.Add(static_cast(Value & 0xff)); + Buffer.Add(static_cast((Value >> 8) & 0xff)); + } + + void AppendLe32(TArray& Buffer, const uint32 Value) + { + Buffer.Add(static_cast(Value & 0xff)); + Buffer.Add(static_cast((Value >> 8) & 0xff)); + Buffer.Add(static_cast((Value >> 16) & 0xff)); + Buffer.Add(static_cast((Value >> 24) & 0xff)); + } + + TArray MakeSilentMonoPcm16Wav(const int32 SampleRateHz, const int32 DurationMs) + { + const int32 ClampedSampleRate = FMath::Max(SampleRateHz, 8000); + const int32 ClampedDurationMs = FMath::Max(DurationMs, 100); + const uint32 SampleCount = static_cast((static_cast(ClampedSampleRate) * ClampedDurationMs) / 1000); + const uint16 ChannelCount = 1; + const uint16 BitsPerSample = 16; + const uint16 BlockAlign = static_cast(ChannelCount * (BitsPerSample / 8)); + const uint32 ByteRate = static_cast(ClampedSampleRate) * BlockAlign; + const uint32 DataSize = SampleCount * BlockAlign; + const uint32 RiffSize = 36 + DataSize; + + TArray Buffer; + Buffer.Reserve(static_cast(44 + DataSize)); + Buffer.Append(reinterpret_cast("RIFF"), 4); + AppendLe32(Buffer, RiffSize); + Buffer.Append(reinterpret_cast("WAVE"), 4); + Buffer.Append(reinterpret_cast("fmt "), 4); + AppendLe32(Buffer, 16); + AppendLe16(Buffer, 1); + AppendLe16(Buffer, ChannelCount); + AppendLe32(Buffer, static_cast(ClampedSampleRate)); + AppendLe32(Buffer, ByteRate); + AppendLe16(Buffer, BlockAlign); + AppendLe16(Buffer, BitsPerSample); + Buffer.Append(reinterpret_cast("data"), 4); + AppendLe32(Buffer, DataSize); + Buffer.AddZeroed(static_cast(DataSize)); + return Buffer; + } + + FHyperTwistVoiceProfileSummary MakeVoiceProfile( + const TCHAR* VoiceProfileId, + const TCHAR* VoiceName, + const TCHAR* LanguageCode, + const TCHAR* LanguageFamily, + const TCHAR* RegionCode, + const TCHAR* LanguageNameEnglish, + const TCHAR* Quality, + const int32 NumSpeakers, + const TCHAR* ModelCardRelativePath, + const TArray& Aliases + ) + { + FHyperTwistVoiceProfileSummary Profile; + Profile.VoiceProfileId = VoiceProfileId; + Profile.VoiceName = VoiceName; + Profile.LanguageCode = LanguageCode; + Profile.LanguageFamily = LanguageFamily; + Profile.RegionCode = RegionCode; + Profile.LanguageNameEnglish = LanguageNameEnglish; + Profile.Quality = Quality; + Profile.NumSpeakers = NumSpeakers; + Profile.ModelCardRelativePath = ModelCardRelativePath; + Profile.Aliases = Aliases; + return Profile; + } + FHyperTwistTrainingSourceAttribution MakeMethodDrillCleanRoomAttribution() { FHyperTwistTrainingSourceAttribution Attribution; @@ -716,6 +788,119 @@ FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechService return Health; } +TArray UHyperTwistContractLibrary::MakeSampleVoiceProfiles() +{ + return { + HyperTwistContractLibraryInternal::MakeVoiceProfile( + TEXT("en_US-lessac-medium"), + TEXT("Lessac"), + TEXT("en_US"), + TEXT("en"), + TEXT("US"), + TEXT("English"), + TEXT("medium"), + 1, + TEXT("en/en_US/lessac/medium/MODEL_CARD"), + {TEXT("en-us-lessac")} + ), + HyperTwistContractLibraryInternal::MakeVoiceProfile( + TEXT("en_GB-alan-medium"), + TEXT("Alan"), + TEXT("en_GB"), + TEXT("en"), + TEXT("GB"), + TEXT("English"), + TEXT("medium"), + 1, + TEXT("en/en_GB/alan/medium/MODEL_CARD"), + {TEXT("en-gb-alan")} + ), + HyperTwistContractLibraryInternal::MakeVoiceProfile( + TEXT("de_DE-thorsten-medium"), + TEXT("Thorsten"), + TEXT("de_DE"), + TEXT("de"), + TEXT("DE"), + TEXT("German"), + TEXT("medium"), + 1, + TEXT("de/de_DE/thorsten/medium/MODEL_CARD"), + {TEXT("de-de-thorsten")} + ) + }; +} + +FHyperTwistNarrationSynthesisRequest UHyperTwistContractLibrary::MakeSampleNarrationSynthesisRequest() +{ + FHyperTwistNarrationSynthesisRequest Request; + Request.RequestId = TEXT("narration_01"); + Request.NarrationContractId = TEXT("embodied-companion-narration"); + Request.ServiceLaneId = TEXT("voice/local-sidecar"); + Request.OutputRouteId = TEXT("companion/narration/local"); + Request.VoiceProfileId = TEXT("en_US-lessac-medium"); + Request.LanguageCode = TEXT("en_US"); + Request.ScriptText = TEXT("Good work. Keep your eyes on the next pair."); + Request.SubtitleSeedText = Request.ScriptText; + Request.SpeakerId = 0; + Request.LengthScale = 1.0f; + Request.NoiseScale = 0.667f; + Request.NoiseW = 0.8f; + Request.SentenceSilenceSeconds = 0.2f; + Request.bPreferCuda = false; + Request.AudioEncodingHint = TEXT("wav-pcm16-mono"); + return Request; +} + +FHyperTwistNarrationSynthesisResult UHyperTwistContractLibrary::MakeMockNarrationSynthesisResult() +{ + const FHyperTwistNarrationSynthesisRequest Request = MakeSampleNarrationSynthesisRequest(); + + FHyperTwistNarrationSynthesisResult Result; + Result.RequestId = Request.RequestId; + Result.NarrationContractId = Request.NarrationContractId; + Result.ServiceLaneId = Request.ServiceLaneId; + Result.OutputRouteId = Request.OutputRouteId; + Result.VoiceProfileId = Request.VoiceProfileId; + Result.LanguageCode = Request.LanguageCode; + Result.SubtitleText = Request.SubtitleSeedText; + Result.AudioEncoding = TEXT("wav-pcm16-mono"); + Result.SampleRateHz = 16000; + Result.ChannelCount = 1; + Result.DurationMs = 1400; + Result.AppliedLengthScale = Request.LengthScale; + Result.AppliedNoiseScale = Request.NoiseScale; + Result.AppliedNoiseW = Request.NoiseW; + Result.AppliedSentenceSilenceSeconds = Request.SentenceSilenceSeconds; + Result.AudioBytes = HyperTwistContractLibraryInternal::MakeSilentMonoPcm16Wav( + Result.SampleRateHz, + Result.DurationMs + ); + return Result; +} + +FHyperTwistVoiceServiceHealth UHyperTwistContractLibrary::MakeMockVoiceServiceHealth() +{ + FHyperTwistVoiceServiceHealth Health; + Health.ProviderLabel = TEXT("mock"); + Health.ServiceVersion = TEXT("mock-voice/v1"); + Health.ServiceEndpoint = TEXT("in-process://mock"); + Health.Capabilities = { + TEXT("voiceCatalog"), + TEXT("offlineNarration"), + TEXT("wavOutput"), + TEXT("subtitleSeed"), + TEXT("localProfileCatalog") + }; + + for (const FHyperTwistVoiceProfileSummary& Profile : MakeSampleVoiceProfiles()) + { + Health.SupportedVoiceProfileIds.Add(Profile.VoiceProfileId); + } + + Health.bReady = true; + return Health; +} + FHyperTwistContentPack UHyperTwistContractLibrary::MakeSampleAlgTrainerContentPack() { return UHyperTwistTrainingCatalogLibrary::MakeAlgTrainerStarterContentPack(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVoiceClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVoiceClient.cpp new file mode 100644 index 0000000..a2d6057 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVoiceClient.cpp @@ -0,0 +1,440 @@ +#include "HyperTwistRecognition/HyperTwistVoiceClient.h" + +#include "Dom/JsonObject.h" +#include "Http.h" +#include "HttpManager.h" +#include "HttpModule.h" +#include "Interfaces/IHttpRequest.h" +#include "Interfaces/IHttpResponse.h" +#include "JsonObjectConverter.h" +#include "Serialization/JsonSerializer.h" +#include "Serialization/JsonWriter.h" + +namespace HyperTwistHttpVoiceClientInternal +{ + template + bool SerializeStruct(const TStruct& Value, FString& OutJson) + { + return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0); + } + + template + bool DeserializeStruct(const FString& Json, TStruct& OutValue) + { + return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0); + } + + FString BuildUrl(const FString& BaseUrl, const FString& Path) + { + if (BaseUrl.IsEmpty()) + { + return FString(); + } + + const FString TrimmedBase = BaseUrl.EndsWith(TEXT("/")) ? BaseUrl.LeftChop(1) : BaseUrl; + if (Path.IsEmpty()) + { + return TrimmedBase; + } + + const FString TrimmedPath = Path.StartsWith(TEXT("/")) ? Path.RightChop(1) : Path; + return FString::Printf(TEXT("%s/%s"), *TrimmedBase, *TrimmedPath); + } + + struct FHttpResponseState + { + bool bCompleted = false; + bool bSucceeded = false; + int32 StatusCode = 0; + FString ResponseBody; + TArray ResponseBytes; + FString Error; + }; + + bool ExecuteRequest( + const UHyperTwistHttpVoiceClient& Client, + const FString& Verb, + const FString& Url, + const FString& AcceptHeader, + const FString& ContentType, + const FString& RequestBody, + FHttpResponseState& OutResponse) + { + OutResponse = FHttpResponseState(); + + if (Url.IsEmpty()) + { + OutResponse.Error = TEXT("service-base-url-missing"); + return false; + } + + const TSharedRef Request = FHttpModule::Get().CreateRequest(); + Request->SetURL(Url); + Request->SetVerb(Verb); + Request->SetHeader(TEXT("Accept"), AcceptHeader); + if (!ContentType.IsEmpty()) + { + Request->SetHeader(TEXT("Content-Type"), ContentType); + } + + if (!Client.AuthorizationToken.IsEmpty()) + { + Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *Client.AuthorizationToken)); + } + + if (!RequestBody.IsEmpty()) + { + Request->SetContentAsString(RequestBody); + } + + Request->OnProcessRequestComplete().BindLambda( + [&OutResponse](FHttpRequestPtr, FHttpResponsePtr Response, bool bWasSuccessful) + { + OutResponse.bCompleted = true; + + if (Response.IsValid()) + { + OutResponse.StatusCode = Response->GetResponseCode(); + OutResponse.ResponseBody = Response->GetContentAsString(); + OutResponse.ResponseBytes = Response->GetContent(); + } + + if (!bWasSuccessful) + { + OutResponse.Error = TEXT("request-failed"); + return; + } + + if (!Response.IsValid()) + { + OutResponse.Error = TEXT("response-missing"); + return; + } + + if (!EHttpResponseCodes::IsOk(OutResponse.StatusCode)) + { + OutResponse.Error = FString::Printf(TEXT("http-%d"), OutResponse.StatusCode); + return; + } + + OutResponse.bSucceeded = true; + }); + + if (!Request->ProcessRequest()) + { + OutResponse.Error = TEXT("request-start-failed"); + return false; + } + + const double TimeoutSeconds = FMath::Max(static_cast(Client.RequestTimeoutSeconds), 0.1); + const double Deadline = FPlatformTime::Seconds() + TimeoutSeconds; + while (!OutResponse.bCompleted && FPlatformTime::Seconds() < Deadline) + { + FHttpModule::Get().GetHttpManager().Tick(0.01f); + FPlatformProcess::Sleep(0.01f); + } + + if (!OutResponse.bCompleted) + { + Request->CancelRequest(); + OutResponse.Error = TEXT("request-timeout"); + return false; + } + + return OutResponse.bSucceeded; + } + + bool ReadLe16(const TArray& Bytes, const int32 Offset, uint16& OutValue) + { + if (Offset < 0 || Offset + 1 >= Bytes.Num()) + { + return false; + } + + OutValue = static_cast(Bytes[Offset]) | (static_cast(Bytes[Offset + 1]) << 8); + return true; + } + + bool ReadLe32(const TArray& Bytes, const int32 Offset, uint32& OutValue) + { + if (Offset < 0 || Offset + 3 >= Bytes.Num()) + { + return false; + } + + OutValue = static_cast(Bytes[Offset]) + | (static_cast(Bytes[Offset + 1]) << 8) + | (static_cast(Bytes[Offset + 2]) << 16) + | (static_cast(Bytes[Offset + 3]) << 24); + return true; + } + + bool ParseWavMetadata( + const TArray& Bytes, + int32& OutSampleRateHz, + int32& OutChannelCount, + int32& OutDurationMs) + { + OutSampleRateHz = 0; + OutChannelCount = 0; + OutDurationMs = 0; + + if (Bytes.Num() < 44 + || Bytes[0] != 'R' + || Bytes[1] != 'I' + || Bytes[2] != 'F' + || Bytes[3] != 'F' + || Bytes[8] != 'W' + || Bytes[9] != 'A' + || Bytes[10] != 'V' + || Bytes[11] != 'E') + { + return false; + } + + int32 Offset = 12; + uint16 Channels = 0; + uint32 SampleRate = 0; + uint16 BitsPerSample = 0; + uint32 DataSize = 0; + + while (Offset + 8 <= Bytes.Num()) + { + const ANSICHAR ChunkIdChars[5] = { + static_cast(Bytes[Offset]), + static_cast(Bytes[Offset + 1]), + static_cast(Bytes[Offset + 2]), + static_cast(Bytes[Offset + 3]), + 0 + }; + const FString ChunkId(ANSI_TO_TCHAR(ChunkIdChars)); + + uint32 ChunkSize = 0; + if (!ReadLe32(Bytes, Offset + 4, ChunkSize)) + { + return false; + } + + const int32 ChunkDataOffset = Offset + 8; + if (ChunkDataOffset + static_cast(ChunkSize) > Bytes.Num()) + { + return false; + } + + if (ChunkId == TEXT("fmt ")) + { + uint16 AudioFormat = 0; + if (!ReadLe16(Bytes, ChunkDataOffset, AudioFormat) + || !ReadLe16(Bytes, ChunkDataOffset + 2, Channels) + || !ReadLe32(Bytes, ChunkDataOffset + 4, SampleRate) + || !ReadLe16(Bytes, ChunkDataOffset + 14, BitsPerSample)) + { + return false; + } + + if (AudioFormat != 1) + { + return false; + } + } + else if (ChunkId == TEXT("data")) + { + DataSize = ChunkSize; + } + + Offset = ChunkDataOffset + static_cast(ChunkSize); + if ((ChunkSize % 2U) != 0U) + { + ++Offset; + } + } + + if (Channels <= 0 || SampleRate <= 0 || BitsPerSample <= 0 || DataSize <= 0) + { + return false; + } + + const int32 BytesPerSampleFrame = Channels * (BitsPerSample / 8); + if (BytesPerSampleFrame <= 0) + { + return false; + } + + const double FrameCount = static_cast(DataSize) / static_cast(BytesPerSampleFrame); + OutSampleRateHz = static_cast(SampleRate); + OutChannelCount = static_cast(Channels); + OutDurationMs = FMath::Max( + 0, + FMath::RoundToInt(static_cast((FrameCount * 1000.0) / static_cast(SampleRate))) + ); + return true; + } +} + +TArray UHyperTwistHttpVoiceClient::ListVoiceProfiles() const +{ + HyperTwistHttpVoiceClientInternal::FHttpResponseState Response; + if (!HyperTwistHttpVoiceClientInternal::ExecuteRequest( + *this, + TEXT("GET"), + HyperTwistHttpVoiceClientInternal::BuildUrl(ServiceBaseUrl, ProfilesPath), + TEXT("application/json"), + FString(), + FString(), + Response)) + { + LastTransportError = Response.Error; + return {}; + } + + FHyperTwistVoiceProfileCatalog Catalog; + if (!HyperTwistHttpVoiceClientInternal::DeserializeStruct(Response.ResponseBody, Catalog) + || !Catalog.IsStructurallyValid()) + { + LastTransportError = TEXT("invalid-provider-response"); + return {}; + } + + LastTransportError.Reset(); + return Catalog.Profiles; +} + +FHyperTwistNarrationSynthesisResult UHyperTwistHttpVoiceClient::SynthesizeNarration( + const FHyperTwistNarrationSynthesisRequest& Request) +{ + FHyperTwistNarrationSynthesisResult Result; + Result.RequestId = Request.RequestId; + Result.NarrationContractId = Request.NarrationContractId; + Result.ServiceLaneId = Request.ServiceLaneId; + Result.OutputRouteId = Request.OutputRouteId; + Result.VoiceProfileId = Request.VoiceProfileId; + Result.LanguageCode = Request.LanguageCode; + Result.SubtitleText = Request.SubtitleSeedText.IsEmpty() ? Request.ScriptText : Request.SubtitleSeedText; + Result.AudioEncoding = TEXT("wav-pcm16-mono"); + Result.AppliedLengthScale = Request.LengthScale; + Result.AppliedNoiseScale = Request.NoiseScale; + Result.AppliedNoiseW = Request.NoiseW; + Result.AppliedSentenceSilenceSeconds = Request.SentenceSilenceSeconds; + + if (!Request.IsStructurallyValid()) + { + Result.Warnings.Add(TEXT("narration-request-invalid")); + return Result; + } + + FString RequestJson; + if (!HyperTwistHttpVoiceClientInternal::SerializeStruct(Request, RequestJson)) + { + LastTransportError = TEXT("request-serialize-failed"); + Result.Warnings.Add(LastTransportError); + return Result; + } + + HyperTwistHttpVoiceClientInternal::FHttpResponseState Response; + if (!HyperTwistHttpVoiceClientInternal::ExecuteRequest( + *this, + TEXT("POST"), + HyperTwistHttpVoiceClientInternal::BuildUrl(ServiceBaseUrl, SynthesizePath), + TEXT("audio/wav"), + TEXT("application/json"), + RequestJson, + Response)) + { + LastTransportError = Response.Error; + Result.Warnings.Add(LastTransportError); + return Result; + } + + int32 SampleRateHz = 0; + int32 ChannelCount = 0; + int32 DurationMs = 0; + if (!HyperTwistHttpVoiceClientInternal::ParseWavMetadata( + Response.ResponseBytes, + SampleRateHz, + ChannelCount, + DurationMs)) + { + LastTransportError = TEXT("invalid-provider-audio"); + Result.Warnings.Add(LastTransportError); + return Result; + } + + Result.SampleRateHz = SampleRateHz; + Result.ChannelCount = ChannelCount; + Result.DurationMs = DurationMs; + Result.AudioBytes = Response.ResponseBytes; + LastTransportError.Reset(); + return Result; +} + +FHyperTwistVoiceServiceHealth UHyperTwistHttpVoiceClient::GetVoiceServiceHealth() const +{ + FHyperTwistVoiceServiceHealth Health; + Health.ProviderLabel = ProviderLabel; + Health.ServiceVersion = TEXT("provider-unavailable/v1"); + Health.ServiceEndpoint = ServiceBaseUrl; + Health.Capabilities = { + TEXT("transport:http"), + TEXT("provider-backed-voice"), + TEXT("voiceCatalog"), + TEXT("offlineNarration"), + TEXT("wavOutput"), + TEXT("subtitleSeed") + }; + Health.bReady = false; + + HyperTwistHttpVoiceClientInternal::FHttpResponseState Response; + if (!HyperTwistHttpVoiceClientInternal::ExecuteRequest( + *this, + TEXT("GET"), + HyperTwistHttpVoiceClientInternal::BuildUrl(ServiceBaseUrl, HealthPath), + TEXT("application/json"), + FString(), + FString(), + Response)) + { + LastTransportError = Response.Error; + Health.LastError = LastTransportError; + return Health; + } + + if (!HyperTwistHttpVoiceClientInternal::DeserializeStruct(Response.ResponseBody, Health) + || !Health.IsStructurallyValid()) + { + LastTransportError = TEXT("invalid-provider-response"); + Health = FHyperTwistVoiceServiceHealth(); + Health.ProviderLabel = ProviderLabel; + Health.ServiceVersion = TEXT("provider-unavailable/v1"); + Health.ServiceEndpoint = ServiceBaseUrl; + Health.Capabilities = { + TEXT("transport:http"), + TEXT("provider-backed-voice"), + TEXT("voiceCatalog"), + TEXT("offlineNarration"), + TEXT("wavOutput"), + TEXT("subtitleSeed") + }; + Health.bReady = false; + Health.LastError = LastTransportError; + return Health; + } + + if (Health.ProviderLabel.IsEmpty()) + { + Health.ProviderLabel = ProviderLabel; + } + if (Health.ServiceEndpoint.IsEmpty()) + { + Health.ServiceEndpoint = ServiceBaseUrl; + } + + Health.Capabilities.AddUnique(TEXT("transport:http")); + Health.Capabilities.AddUnique(TEXT("provider-backed-voice")); + Health.Capabilities.AddUnique(TEXT("voiceCatalog")); + Health.Capabilities.AddUnique(TEXT("offlineNarration")); + Health.Capabilities.AddUnique(TEXT("wavOutput")); + Health.Capabilities.AddUnique(TEXT("subtitleSeed")); + LastTransportError.Reset(); + Health.LastError.Reset(); + return Health; +} diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVoiceClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVoiceClient.cpp new file mode 100644 index 0000000..1bbbbda --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVoiceClient.cpp @@ -0,0 +1,68 @@ +#include "HyperTwistRecognition/HyperTwistVoiceClient.h" + +#include "HyperTwistBootstrap/HyperTwistContractLibrary.h" + +TArray UHyperTwistMockVoiceClient::ListVoiceProfiles() const +{ + return UHyperTwistContractLibrary::MakeSampleVoiceProfiles(); +} + +FHyperTwistNarrationSynthesisResult UHyperTwistMockVoiceClient::SynthesizeNarration( + const FHyperTwistNarrationSynthesisRequest& Request +) +{ + FHyperTwistNarrationSynthesisResult Result = UHyperTwistContractLibrary::MakeMockNarrationSynthesisResult(); + Result.RequestId = Request.RequestId; + Result.NarrationContractId = Request.NarrationContractId; + Result.ServiceLaneId = Request.ServiceLaneId; + Result.OutputRouteId = Request.OutputRouteId; + Result.VoiceProfileId = Request.VoiceProfileId; + Result.LanguageCode = Request.LanguageCode; + Result.SubtitleText = Request.SubtitleSeedText.IsEmpty() ? Request.ScriptText : Request.SubtitleSeedText; + Result.AppliedLengthScale = Request.LengthScale; + Result.AppliedNoiseScale = Request.NoiseScale; + Result.AppliedNoiseW = Request.NoiseW; + Result.AppliedSentenceSilenceSeconds = Request.SentenceSilenceSeconds; + + if (!Request.IsStructurallyValid()) + { + Result = FHyperTwistNarrationSynthesisResult(); + Result.RequestId = Request.RequestId; + Result.NarrationContractId = Request.NarrationContractId; + Result.ServiceLaneId = Request.ServiceLaneId; + Result.OutputRouteId = Request.OutputRouteId; + Result.VoiceProfileId = Request.VoiceProfileId; + Result.LanguageCode = Request.LanguageCode; + Result.Warnings.Add(TEXT("narration-request-invalid")); + return Result; + } + + bool bKnownProfile = false; + for (const FHyperTwistVoiceProfileSummary& Profile : UHyperTwistContractLibrary::MakeSampleVoiceProfiles()) + { + if (Profile.VoiceProfileId.Equals(Request.VoiceProfileId, ESearchCase::IgnoreCase)) + { + bKnownProfile = true; + break; + } + } + + if (!bKnownProfile) + { + Result.Warnings.Add(TEXT("voice-profile-not-found")); + } + + return Result; +} + +FHyperTwistVoiceServiceHealth UHyperTwistMockVoiceClient::GetVoiceServiceHealth() const +{ + FHyperTwistVoiceServiceHealth Health = UHyperTwistContractLibrary::MakeMockVoiceServiceHealth(); + Health.ProviderLabel = TEXT("mock"); + Health.ServiceEndpoint = TEXT("in-process://mock"); + Health.Capabilities.AddUnique(TEXT("offlineNarration")); + Health.Capabilities.AddUnique(TEXT("voiceCatalog")); + Health.Capabilities.AddUnique(TEXT("wavOutput")); + Health.Capabilities.AddUnique(TEXT("subtitleSeed")); + return Health; +} diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h index 9e2f176..29e6f1f 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistBootstrap/HyperTwistContractLibrary.h @@ -66,6 +66,18 @@ public: UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") static FHyperTwistSpeechServiceHealth MakeMockSpeechServiceHealth(); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static TArray MakeSampleVoiceProfiles(); + + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistNarrationSynthesisRequest MakeSampleNarrationSynthesisRequest(); + + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistNarrationSynthesisResult MakeMockNarrationSynthesisResult(); + + UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition") + static FHyperTwistVoiceServiceHealth MakeMockVoiceServiceHealth(); + UFUNCTION(BlueprintPure, Category = "HyperTwist|Training") static FHyperTwistContentPack MakeSampleAlgTrainerContentPack(); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h index 267ed99..bd4bd31 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h @@ -930,3 +930,254 @@ struct FHyperTwistSpeechServiceHealth UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") bool bReady = true; }; + +USTRUCT(BlueprintType) +struct FHyperTwistVoiceProfileSummary +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString VoiceProfileId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString VoiceName; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageCode = TEXT("en_US"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageFamily = TEXT("en"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString RegionCode = TEXT("US"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageNameEnglish = TEXT("English"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString Quality = TEXT("medium"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 NumSpeakers = 1; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ModelCardRelativePath; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Aliases; + + bool IsStructurallyValid() const + { + return !VoiceProfileId.IsEmpty() + && !VoiceName.IsEmpty() + && !LanguageCode.IsEmpty() + && !LanguageFamily.IsEmpty() + && !RegionCode.IsEmpty() + && !LanguageNameEnglish.IsEmpty() + && !Quality.IsEmpty() + && NumSpeakers > 0; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistVoiceProfileCatalog +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Profiles; + + bool IsStructurallyValid() const + { + if (Profiles.Num() <= 0) + { + return false; + } + + for (const FHyperTwistVoiceProfileSummary& Profile : Profiles) + { + if (!Profile.IsStructurallyValid()) + { + return false; + } + } + + return true; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistNarrationSynthesisRequest +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString RequestId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString NarrationContractId = TEXT("embodied-companion-narration"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceLaneId = TEXT("voice/local-sidecar"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString OutputRouteId = TEXT("companion/narration/local"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString VoiceProfileId = TEXT("en_US-lessac-medium"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageCode = TEXT("en_US"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ScriptText; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SubtitleSeedText; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SpeakerId = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float LengthScale = 1.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float NoiseScale = 0.667f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float NoiseW = 0.8f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float SentenceSilenceSeconds = 0.2f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bPreferCuda = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString AudioEncodingHint = TEXT("wav-pcm16-mono"); + + bool IsStructurallyValid() const + { + return !RequestId.IsEmpty() + && !NarrationContractId.IsEmpty() + && !ServiceLaneId.IsEmpty() + && !OutputRouteId.IsEmpty() + && !VoiceProfileId.IsEmpty() + && !LanguageCode.IsEmpty() + && !ScriptText.IsEmpty() + && !AudioEncodingHint.IsEmpty() + && LengthScale > 0.0f + && NoiseScale >= 0.0f + && NoiseW >= 0.0f + && SentenceSilenceSeconds >= 0.0f; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistNarrationSynthesisResult +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString RequestId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString NarrationContractId = TEXT("embodied-companion-narration"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceLaneId = TEXT("voice/local-sidecar"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString OutputRouteId = TEXT("companion/narration/local"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString VoiceProfileId = TEXT("en_US-lessac-medium"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LanguageCode = TEXT("en_US"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SubtitleText; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString AudioEncoding = TEXT("wav-pcm16-mono"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SampleRateHz = 16000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 ChannelCount = 1; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 DurationMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float AppliedLengthScale = 1.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float AppliedNoiseScale = 0.667f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float AppliedNoiseW = 0.8f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float AppliedSentenceSilenceSeconds = 0.2f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray AudioBytes; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Warnings; + + bool IsStructurallyValid() const + { + return !RequestId.IsEmpty() + && !NarrationContractId.IsEmpty() + && !ServiceLaneId.IsEmpty() + && !OutputRouteId.IsEmpty() + && !VoiceProfileId.IsEmpty() + && !LanguageCode.IsEmpty() + && !SubtitleText.IsEmpty() + && !AudioEncoding.IsEmpty() + && SampleRateHz > 0 + && ChannelCount > 0 + && DurationMs >= 0 + && AudioBytes.Num() > 0 + && AppliedLengthScale > 0.0f + && AppliedNoiseScale >= 0.0f + && AppliedNoiseW >= 0.0f + && AppliedSentenceSilenceSeconds >= 0.0f; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistVoiceServiceHealth +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ProviderLabel = TEXT("mock"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceVersion = TEXT("mock-voice/v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ServiceEndpoint; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Capabilities; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray SupportedVoiceProfileIds; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastError; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bReady = true; + + bool IsStructurallyValid() const + { + return !ProviderLabel.IsEmpty() && !ServiceVersion.IsEmpty(); + } +}; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVoiceClient.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVoiceClient.h new file mode 100644 index 0000000..09ae4a9 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVoiceClient.h @@ -0,0 +1,85 @@ +#pragma once + +#include "CoreMinimal.h" +#include "UObject/Interface.h" +#include "UObject/Object.h" +#include "HyperTwistRecognition/HyperTwistRecognitionTypes.h" +#include "HyperTwistVoiceClient.generated.h" + +UINTERFACE(BlueprintType) +class UNREALHYPERTWIST_API UHyperTwistVoiceClient : public UInterface +{ + GENERATED_BODY() +}; + +class UNREALHYPERTWIST_API IHyperTwistVoiceClient +{ + GENERATED_BODY() + +public: + virtual TArray ListVoiceProfiles() const = 0; + virtual FHyperTwistNarrationSynthesisResult SynthesizeNarration( + const FHyperTwistNarrationSynthesisRequest& Request + ) = 0; + virtual FHyperTwistVoiceServiceHealth GetVoiceServiceHealth() const = 0; +}; + +UCLASS(BlueprintType) +class UNREALHYPERTWIST_API UHyperTwistMockVoiceClient : public UObject, public IHyperTwistVoiceClient +{ + GENERATED_BODY() + +public: + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual TArray ListVoiceProfiles() const override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistNarrationSynthesisResult SynthesizeNarration( + const FHyperTwistNarrationSynthesisRequest& Request + ) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistVoiceServiceHealth GetVoiceServiceHealth() const override; +}; + +UCLASS(BlueprintType, Config = Game, DefaultConfig) +class UNREALHYPERTWIST_API UHyperTwistHttpVoiceClient : public UObject, public IHyperTwistVoiceClient +{ + GENERATED_BODY() + +public: + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString ProviderLabel = TEXT("local-http-sidecar"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString ServiceBaseUrl = TEXT("http://127.0.0.1:8766"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString ProfilesPath = TEXT("/voice/profiles"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString SynthesizePath = TEXT("/voice/narrate"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString HealthPath = TEXT("/voice/health"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + float RequestTimeoutSeconds = 5.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition") + FString AuthorizationToken; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual TArray ListVoiceProfiles() const override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistNarrationSynthesisResult SynthesizeNarration( + const FHyperTwistNarrationSynthesisRequest& Request + ) override; + + UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition") + virtual FHyperTwistVoiceServiceHealth GetVoiceServiceHealth() const override; + +private: + mutable FString LastTransportError; +}; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistPiperPhase6RGLocalNarrationContractTest.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistPiperPhase6RGLocalNarrationContractTest.cpp new file mode 100644 index 0000000..56de539 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistPiperPhase6RGLocalNarrationContractTest.cpp @@ -0,0 +1,134 @@ +// Copyright HyperTwist, Inc. All Rights Reserved. + +#include "Misc/AutomationTest.h" + +#include "HyperTwistBootstrap/HyperTwistContractLibrary.h" +#include "HyperTwistRecognition/HyperTwistVoiceClient.h" +#include "HyperTwistTraining/HyperTwistTrainingRuntimeLibrary.h" + +#if WITH_AUTOMATION_TESTS + +namespace HyperTwistPiperPhase6RGTestInternal +{ + UHyperTwistMockVoiceClient* MakeMockVoiceClient() + { + return NewObject(GetTransientPackage()); + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistPiperPhase6RGNarrationRequestTest, + "HyperTwist.Permissive.Piper.Phase6R.G.NarrationRequest", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistPiperPhase6RGNarrationRequestTest::RunTest(const FString& Parameters) +{ + FHyperTwistTrainingCompanionNarrationContract NarrationContract; + TestTrue( + TEXT("The bundled embodied companion narration contract must be available."), + UHyperTwistTrainingRuntimeLibrary::TryGetBundledEmbodiedCompanionNarrationContract( + TEXT("embodied-companion-narration"), + NarrationContract + ) + ); + TestTrue(TEXT("The bundled narration contract must be structurally valid."), NarrationContract.IsStructurallyValid()); + TestTrue( + TEXT("The narration contract must preserve subtitle synchronization in the output surface tags."), + NarrationContract.OutputSurfaceTags.Contains(TEXT("subtitle-sync")) + ); + + UHyperTwistMockVoiceClient* VoiceClient = HyperTwistPiperPhase6RGTestInternal::MakeMockVoiceClient(); + TestNotNull(TEXT("The mock voice client must be constructed for Phase 6R-G."), VoiceClient); + if (VoiceClient == nullptr) + { + return false; + } + + const TArray Profiles = VoiceClient->ListVoiceProfiles(); + TestTrue(TEXT("The bounded piper route must expose a non-empty local voice catalog."), Profiles.Num() >= 3); + TestTrue( + TEXT("The local voice catalog must contain the bounded Lessac profile."), + Profiles.ContainsByPredicate([](const FHyperTwistVoiceProfileSummary& Profile) + { + return Profile.VoiceProfileId == TEXT("en_US-lessac-medium"); + }) + ); + + const FHyperTwistNarrationSynthesisRequest Request = UHyperTwistContractLibrary::MakeSampleNarrationSynthesisRequest(); + TestTrue(TEXT("The sample narration request must be structurally valid."), Request.IsStructurallyValid()); + TestEqual( + TEXT("The request must anchor on the bundled embodied companion narration contract."), + Request.NarrationContractId, + TEXT("embodied-companion-narration") + ); + TestEqual(TEXT("The bounded first slice must stay on the local voice sidecar lane."), Request.ServiceLaneId, TEXT("voice/local-sidecar")); + TestEqual(TEXT("The bounded first slice must keep the local companion narration route."), Request.OutputRouteId, TEXT("companion/narration/local")); + TestEqual(TEXT("The bounded first slice must use WAV mono output."), Request.AudioEncodingHint, TEXT("wav-pcm16-mono")); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistPiperPhase6RGSynthesisResultTest, + "HyperTwist.Permissive.Piper.Phase6R.G.SynthesisResult", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistPiperPhase6RGSynthesisResultTest::RunTest(const FString& Parameters) +{ + UHyperTwistMockVoiceClient* VoiceClient = HyperTwistPiperPhase6RGTestInternal::MakeMockVoiceClient(); + TestNotNull(TEXT("The mock voice client must be constructed for Phase 6R-G."), VoiceClient); + if (VoiceClient == nullptr) + { + return false; + } + + const FHyperTwistNarrationSynthesisRequest Request = UHyperTwistContractLibrary::MakeSampleNarrationSynthesisRequest(); + const FHyperTwistNarrationSynthesisResult Result = VoiceClient->SynthesizeNarration(Request); + + TestTrue(TEXT("The bounded narration synthesis result must be structurally valid."), Result.IsStructurallyValid()); + TestEqual(TEXT("The result must preserve the request id."), Result.RequestId, Request.RequestId); + TestEqual(TEXT("The result must preserve the voice profile id."), Result.VoiceProfileId, Request.VoiceProfileId); + TestEqual( + TEXT("The result must preserve the subtitle seed text for the local narration sidecar."), + Result.SubtitleText, + Request.SubtitleSeedText + ); + TestEqual(TEXT("The result must stay WAV mono."), Result.AudioEncoding, TEXT("wav-pcm16-mono")); + TestEqual(TEXT("The bounded local sidecar must keep the donor-grounded sample rate."), Result.SampleRateHz, 16000); + TestEqual(TEXT("The bounded local sidecar must keep mono output."), Result.ChannelCount, 1); + TestTrue(TEXT("The bounded local sidecar must return synthesized audio bytes."), Result.AudioBytes.Num() > 44); + TestTrue(TEXT("The bounded local sidecar must report positive duration."), Result.DurationMs > 0); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistPiperPhase6RGServiceHealthTest, + "HyperTwist.Permissive.Piper.Phase6R.G.ServiceHealth", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistPiperPhase6RGServiceHealthTest::RunTest(const FString& Parameters) +{ + UHyperTwistMockVoiceClient* VoiceClient = HyperTwistPiperPhase6RGTestInternal::MakeMockVoiceClient(); + TestNotNull(TEXT("The mock voice client must be constructed for Phase 6R-G."), VoiceClient); + if (VoiceClient == nullptr) + { + return false; + } + + const FHyperTwistVoiceServiceHealth Health = VoiceClient->GetVoiceServiceHealth(); + TestTrue(TEXT("The bounded voice service health must be structurally valid."), Health.IsStructurallyValid()); + TestTrue(TEXT("The bounded voice service health must report ready."), Health.bReady); + TestTrue(TEXT("The bounded voice service must expose voice catalog capability."), Health.Capabilities.Contains(TEXT("voiceCatalog"))); + TestTrue(TEXT("The bounded voice service must expose offline narration capability."), Health.Capabilities.Contains(TEXT("offlineNarration"))); + TestTrue(TEXT("The bounded voice service must expose WAV output capability."), Health.Capabilities.Contains(TEXT("wavOutput"))); + TestTrue(TEXT("The bounded voice service must expose subtitle-seed capability."), Health.Capabilities.Contains(TEXT("subtitleSeed"))); + TestTrue( + TEXT("The bounded voice service must advertise the retained Lessac voice profile."), + Health.SupportedVoiceProfileIds.Contains(TEXT("en_US-lessac-medium")) + ); + return true; +} + +#endif diff --git a/docs/REPO_LICENSE_TRACKING.md b/docs/REPO_LICENSE_TRACKING.md index 5ad7644..4583355 100644 --- a/docs/REPO_LICENSE_TRACKING.md +++ b/docs/REPO_LICENSE_TRACKING.md @@ -136,8 +136,12 @@ Status update on `2026-05-21`: - the bounded permissive `Phase 6R-F` `SYSTRAN/faster-whisper` transcription-service orchestration packet is now landed in current code - `SYSTRAN/faster-whisper` remains partially incorporated -- the current next bounded move is a source-backed `Phase 6R-G` `rhasspy/piper` - preparation/control pass for a bounded voice-output sidecar seam, not a new restrictive packet +- the generic source-backed `Phase 6R-G` `rhasspy/piper` control pass is now consumed +- the bounded permissive `Phase 6R-G` `rhasspy/piper` local narration sidecar packet is now + landed in current code +- `rhasspy/piper` remains partially incorporated +- the current next bounded move is a source-backed `Phase 6R-H` `coqui-ai/TTS` + preparation/control pass for a richer bounded voice-output seam, not a new restrictive packet by default - use the repo-row README census, the portfolio standing refresh backfill, and the `2R-A` ownership contract for the current queue after that correction @@ -145,12 +149,12 @@ Status update on `2026-05-21`: This file now also preserves the current truth that future models must not lose: - current curated HyperTwist shallow-eval set: `75` repos -- currently verified live/implemented in checked `UnrealHyperTwist` surfaces: `30` -- permissive live lanes: `19` +- currently verified live/implemented in checked `UnrealHyperTwist` surfaces: `31` +- permissive live lanes: `20` - boundary-sensitive live lanes: `6` - restrictive live lanes: `5` -The nineteen permissive live lanes are: +The twenty permissive live lanes are: - `Aarav2709/KubeTimr` - `HactarCE/Hyperspeedcube` @@ -171,6 +175,7 @@ The nineteen permissive live lanes are: - `roice3/MagicTile` - `ggml-org/whisper.cpp` - `SYSTRAN/faster-whisper` +- `rhasspy/piper` The six boundary-sensitive live lanes are: @@ -1345,6 +1350,7 @@ Decision date: - `2026-04-24` - refreshed on `2026-05-13` +- refreshed on `2026-05-22` Current licensing judgment: @@ -1369,8 +1375,12 @@ Approved working posture: `MIT` code posture - do not over-elevate it into the broader voice strategy; its main strength is lightweight offline narration -- the current next bounded move is a source-backed `Phase 6R-G` preparation/control pass for one - bounded voice-output sidecar seam +- the first bounded local narration sidecar slice is now landed: + - local voice-profile catalog normalization + - narration synthesis request/result contract + - mock and HTTP local sidecar voice-client seam + - local voice service-health exposure +- `rhasspy/piper` remains a bounded donor, not the broad voice-platform owner - keep voice-asset review separate from the code-license judgment ### `screenpipe/screenpipe` diff --git a/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_IMPLEMENTATION_PACKET_2026-05-22.md b/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_IMPLEMENTATION_PACKET_2026-05-22.md new file mode 100644 index 0000000..96157a7 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_IMPLEMENTATION_PACKET_2026-05-22.md @@ -0,0 +1,134 @@ +# HyperTwist Phase 6R-G piper local narration sidecar implementation packet + +Created on `2026-05-22` + +## Status + +- first-party HyperTwist packet +- bounded permissive `Phase 6R-G` implementation slice + +## Purpose + +This packet lands the first narrower bounded slice from the retained `rhasspy/piper` voice-output row. + +The landed slice is: + +- first-party local narration sidecar contract above the existing embodied companion narration lane + +It is not: + +- a broad TTS platform packet +- a downloadable voice-asset packet +- a provider-profile or BYOK packet +- a viseme / gesture runtime packet +- a `coqui-ai/TTS` widening packet + +## Current authority basis + +This implementation packet stands on: + +- `docs/HYPERTWIST_PHASE_0R_PACKET_0R_A_EVALUATION_2026-05-12.md` +- `docs/HYPERTWIST_PHASE_2R_PACKET_2R_A_OWNERSHIP_AND_ACCEPTANCE_CONTRACT_2026-05-13.md` +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/ops/HYPERTWIST_PROVIDER_NEUTRALITY_AND_BYOK_DOCTRINE_2026-05-21.md` +- `docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_PREPARATION_PACKET_2026-05-22.md` + +The retained owner remains: + +- `rhasspy/piper` + +The top-level provider and companion contract owner does not change here: + +- first-party HyperTwist remains the provider-neutral and companion-contract owner +- this packet keeps `rhasspy/piper` bounded to the narrower `A1 / R1 / F2` local narration sidecar slice + +## Landed scope + +The current code now owns a retained local narration sidecar contract through: + +- retained recognition contract types for: + - `FHyperTwistVoiceProfileSummary` + - `FHyperTwistVoiceProfileCatalog` + - `FHyperTwistNarrationSynthesisRequest` + - `FHyperTwistNarrationSynthesisResult` + - `FHyperTwistVoiceServiceHealth` +- sample voice-profile, narration-request, narration-result, and service-health outputs in: + - `UHyperTwistContractLibrary` +- direct-donor voice client seam in: + - `UHyperTwistMockVoiceClient` + - `UHyperTwistHttpVoiceClient` +- first-party config defaults for the bounded local voice sidecar lane in: + - `DefaultGame.ini` +- focused automation coverage in: + - `HyperTwistPiperPhase6RGLocalNarrationContractTest.cpp` + +## Why this is still intentionally bounded + +This packet lands the first voice-output donor seam, but it does not widen into the neighboring retained families. + +Still deferred: + +- downloadable voice/model review +- bundled voice asset shipping +- rich playback runtime ownership +- viseme / gesture runtime integration +- provider-profile and BYOK custody +- broad assistant-platform scope +- `coqui-ai/TTS` + +## Validation + +Build validation: + +- `Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE` + +Focused automation validation: + +- `Automation RunTests HyperTwist.Permissive.Piper.Phase6R.G` + +Regression automation validation: + +- `Automation RunTests HyperTwist.Permissive.FasterWhisper.Phase6R.F` +- `Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.E` +- `Automation RunTests HyperTwist.CleanRoom.CubeDesk` + +Expected covered tests: + +- `NarrationRequest` +- `SynthesisResult` +- `ServiceHealth` +- `TranscriptMetadata` +- `SessionConfig` +- `TranscriptResult` +- existing `CubeDesk` clean-room regression suite + +## Queue effect + +This packet consumes the current `Phase 6R-G` implementation slice. + +`rhasspy/piper` remains only partially incorporated: + +- landed now: + - local voice-profile catalog normalization + - narration synthesis request/result contract + - mock and HTTP local sidecar voice-client seam + - local voice service-health exposure +- still deferred: + - downloadable voice/model review + - bundled voice asset shipping + - rich playback runtime ownership + - viseme / gesture runtime integration + - broad assistant-platform scope + +The next clean move is: + +- source-backed `Phase 6R-H` `coqui-ai/TTS` preparation/control pass + +Keep the future sequencing guards visible: + +- speech-input / voice sidecar set + - keep code-license judgments separate from model, voice, and payload-license review +- `rhasspy/piper` + - keep it bounded to the local narration sidecar seam rather than broad voice-platform ownership +- `coqui-ai/TTS` + - keep future downloadable voice/model review separate from the code-license judgment diff --git a/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_PREPARATION_PACKET_2026-05-22.md b/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_PREPARATION_PACKET_2026-05-22.md new file mode 100644 index 0000000..7172545 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_G_PIPER_LOCAL_NARRATION_SIDECAR_PREPARATION_PACKET_2026-05-22.md @@ -0,0 +1,76 @@ +# HyperTwist Phase 6R-G piper local narration sidecar preparation packet + +Created on `2026-05-22` + +## Status + +- first-party HyperTwist packet +- source-backed permissive `Phase 6R-G` preparation/control slice + +## Purpose + +This packet narrows the retained `rhasspy/piper` row before implementation. + +The retained first slice is: + +- local narration sidecar contract above the already landed first-party companion narration boundary + +It is not: + +- a broad voice-platform packet +- a downloadable voice-asset or model-shipping packet +- a provider-profile or BYOK packet +- a `coqui-ai/TTS` widening packet +- a full companion playback, viseme, or avatar-orchestration packet + +## Source-backed basis + +Primary donor surfaces inspected: + +- `README.md` +- `VOICES.md` +- `src/python_run/README_http.md` +- `src/python_run/piper/http_server.py` +- `src/python_run/piper/voice.py` +- `src/python_run/piper/voices.json` +- `etc/test_voice.onnx.json` + +## Narrowing decision + +The smallest strong donor seam is: + +- voice-profile catalog metadata +- narration synthesis request/result normalization +- local HTTP-sidecar narration transport shape returning WAV output + +The first bounded implementation packet should therefore land only: + +- first-party voice-profile summaries +- first-party narration synthesis request/result contracts +- first-party mock and HTTP voice-client seam +- focused automation around the bounded local narration route + +## Explicit deferrals + +Still deferred after this control pass: + +- downloadable voice/model review +- bundled voice asset shipping +- rich playback runtime ownership +- viseme or gesture runtime integration +- broader voice-policy ownership +- `coqui-ai/TTS` + +## Queue effect + +This preparation packet authorizes one bounded permissive implementation packet: + +- `Phase 6R-G` local narration sidecar contract + +After that packet, move the queue head to: + +- source-backed `Phase 6R-H` `coqui-ai/TTS` preparation/control pass + +Keep the legal guard visible: + +- code-license judgment stays separate from voice/model/payload review diff --git a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md index ab65805..537a2e8 100644 --- a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md +++ b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md @@ -80,9 +80,9 @@ Use these as the current governing docs: The next bounded move is now: -1. source-backed `Phase 6R-G` `rhasspy/piper` preparation/control pass -2. keep it behind the landed `whisper.cpp` and `faster-whisper` STT slices -3. narrow the first widening slice to one bounded voice-output sidecar seam +1. source-backed `Phase 6R-H` `coqui-ai/TTS` preparation/control pass +2. keep it behind the landed `whisper.cpp`, `faster-whisper`, and `piper` bounded slices +3. narrow the first widening slice to one richer bounded voice-output seam 4. keep voice-asset and payload review separate from the code-license judgment ## Memory-specific sequencing rule diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md index 4a7aa8c..c833aa5 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md @@ -182,7 +182,8 @@ repo. | Speech transcript session boundary | Implemented now | landed `whisper.cpp` packet | Current bounded STT truth. | | Provider-backed speech session health and transcript envelopes | Implemented now | first-party speech client surfaces | Current first-party speech seam. | | Python transcription-service orchestration profile | Implemented now | landed `faster-whisper` packet | Current bounded Python STT orchestration truth above the existing speech session boundary. | -| Voice output / narration sidecars | Deep-source grounded retained | `piper`, `coqui-ai/TTS` retained rows | Retained for later bounded work. | +| Local narration sidecar seam | Implemented now | landed `piper` packet | Current bounded offline narration contract, voice catalog, and local sidecar truth. | +| Broad voice output / narration sidecars | Deep-source grounded retained | `piper`, `coqui-ai/TTS` retained remainder | Keep downloadable voice/model review separate from the code-license judgment. | | Broad live microphone shell and downloadable payload shipping | Deep-source grounded retained | speech stack retained remainder | Explicitly deferred. | ### 8. Provider-neutral AI/provider routing diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md index 8edf41d..0603dfa 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md @@ -15,7 +15,7 @@ Canonical discovery surfaces for roadmap interpretation: for skill-layer sequencing - the canonical HyperTwist repo-row portfolio is now treated as `75` rows, not `71` -- currently implemented rows are now `29`, not `20` +- currently implemented rows are now `30`, not `20` - four of the additional current implemented rows are the later-landed restrictive clean-room lanes: - `cubing/alg.js` - `cubing/twisty.js` @@ -31,6 +31,8 @@ Canonical discovery surfaces for roadmap interpretation: - `roice3/MagicTile` - the ninth additional current implemented row is the same-day bounded permissive partial row: - `SYSTRAN/faster-whisper` +- the tenth additional current implemented row is the next-day bounded permissive partial row: + - `rhasspy/piper` - the current next bounded move is **not** another restrictive packet by default - the earlier first-party packet block that was still being treated as next is now confirmed landed: - `d4f4ad3` `Add primary coach orchestration entry lane` @@ -71,8 +73,13 @@ Canonical discovery surfaces for roadmap interpretation: orchestration packet is now landed in current code - `SYSTRAN/faster-whisper` remains partially incorporated; microphone capture, BYOK/profile custody, payload shipping, and voice-output ownership stay deferred -- the current next bounded move is a source-backed `Phase 6R-G` `rhasspy/piper` - preparation/control pass for a bounded voice-output sidecar seam +- the generic source-backed `Phase 6R-G` `rhasspy/piper` control pass is now consumed +- the bounded permissive `Phase 6R-G` `rhasspy/piper` local narration sidecar packet is now landed + in current code +- `rhasspy/piper` remains partially incorporated; downloadable voice/model review, bundled voice + asset shipping, and broad voice-platform ownership stay deferred +- the current next bounded move is a source-backed `Phase 6R-H` `coqui-ai/TTS` + preparation/control pass for a richer bounded voice-output seam - the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than waiting on another first-party packet @@ -100,8 +107,8 @@ Donor-strength rule: - route difficulty changes how retained value may enter the product, not whether it may win technically - current shallow-eval set: `75` repos -- currently verified live/implemented in checked Unreal surfaces: `30` -- permissive live lanes: `19` +- currently verified live/implemented in checked Unreal surfaces: `31` +- permissive live lanes: `20` - boundary-sensitive live lanes: `6` - restrictive live lanes: `5` - the restrictive landed lanes are: @@ -134,8 +141,8 @@ Current routing truth: - active non-live implementation-board rows: `35` - retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9` -The next bounded move is a source-backed `Phase 6R-G` -`rhasspy/piper` preparation/control pass. +The next bounded move is a source-backed `Phase 6R-H` +`coqui-ai/TTS` preparation/control pass. Queue interpretation after that packet: @@ -204,8 +211,21 @@ Queue interpretation after that packet: - downloadable model or payload shipping - TTS or voice-output ownership - broad assistant-platform scope +- `rhasspy/piper` is now landed as a partially incorporated row rather than a generic future + placeholder: + - landed: + - local narration sidecar contract + - voice-profile catalog normalization + - narration synthesis request/result contract + - bounded mock and HTTP voice-client seam + - voice service-health exposure + - still deferred: + - downloadable voice/model review + - bundled voice asset shipping + - rich playback runtime ownership + - viseme / gesture runtime integration + - broad assistant-platform scope - the next queue shape is now: - - `rhasspy/piper` - `coqui-ai/TTS` - keep the speech-input / voice sidecar legal sequencing guard visible: - keep code-license judgments separate from model, voice, and payload-license review