diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp index 921b5df..386dadf 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp @@ -841,6 +841,62 @@ namespace HyperTwistContractLibraryInternal return BrowserShellState; } + FHyperTwistSpeechMicrophoneShellProfile MakeDefaultCoachCommandMicrophoneShellProfile(const FString& ProfileId) + { + FHyperTwistSpeechMicrophoneShellProfile Profile; + Profile.MicrophoneShellProfileId = ProfileId; + Profile.ShellKind = TEXT("microphone-overlay"); + Profile.CaptureMode = TEXT("vad-gated-short-command"); + Profile.StepWindowMs = 3000; + Profile.CaptureWindowMs = 10000; + Profile.KeepWindowMs = 200; + Profile.AudioContextTokens = 0; + Profile.VadThreshold = 0.6f; + Profile.HighPassFrequencyHz = 100.0f; + Profile.bKeepContextBetweenChunks = false; + Profile.bSupportsStreamingPreview = true; + Profile.bSupportsManualCommit = true; + + auto AddPanel = [&Profile]( + const TCHAR* PanelId, + const TCHAR* PanelKind, + const TCHAR* AnchorId, + const bool bVisibleByDefault + ) + { + FHyperTwistSpeechShellPanelLayout Panel; + Panel.PanelId = PanelId; + Panel.PanelKind = PanelKind; + Panel.AnchorId = AnchorId; + Panel.bVisibleByDefault = bVisibleByDefault; + Profile.Panels.Add(Panel); + }; + + auto AddAction = [&Profile]( + const TCHAR* ActionId, + const TCHAR* InputBinding, + const TCHAR* SurfaceId + ) + { + FHyperTwistSpeechShellActionBinding ActionBinding; + ActionBinding.ActionId = ActionId; + ActionBinding.InputBinding = InputBinding; + ActionBinding.SurfaceId = SurfaceId; + Profile.ActionBindings.Add(ActionBinding); + }; + + AddPanel(TEXT("microphone-status-readout"), TEXT("status-readout"), TEXT("top-left"), true); + AddPanel(TEXT("vad-meter"), TEXT("level-meter"), TEXT("left-stack-below-status"), true); + AddPanel(TEXT("transcript-preview"), TEXT("transcript-preview"), TEXT("bottom-left"), true); + AddPanel(TEXT("command-hints"), TEXT("command-hints"), TEXT("top-right"), true); + + AddAction(TEXT("pause-microphone-capture"), TEXT("P"), TEXT("microphone-status-readout")); + AddAction(TEXT("resume-microphone-capture"), TEXT("R"), TEXT("microphone-status-readout")); + AddAction(TEXT("commit-transcript"), TEXT("Enter"), TEXT("transcript-preview")); + AddAction(TEXT("close-speech-session"), TEXT("Escape"), TEXT("session-root")); + return Profile; + } + FHyperTwistVisionCorrectionState MakeClassicCubeCorrectionState( const FHyperTwistVisionSessionConfig& SessionConfig, const FHyperTwistVisionReconstructionSession& ReconstructionSession, @@ -1582,6 +1638,11 @@ FHyperTwistSpeechSessionConfig UHyperTwistContractLibrary::MakeSampleSpeechSessi Config.LanguageMode = TEXT("auto"); Config.TaskKind = TEXT("transcribe"); Config.GrammarProfileId = TEXT("coach-command-grammar-v1"); + Config.MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1"); + Config.MicrophoneShellProfileDefinition = + HyperTwistContractLibraryInternal::MakeDefaultCoachCommandMicrophoneShellProfile( + Config.MicrophoneShellProfileId + ); Config.bEnableVad = true; Config.bEnableWordTimestamps = false; Config.VadPolicy.Threshold = 0.5f; @@ -1662,6 +1723,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechService TEXT("vadSegments"), TEXT("grammarHints"), TEXT("sessioned-mock-client"), + TEXT("microphoneCaptureShell"), + TEXT("streamingPreview"), + TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), TEXT("wordTimestamps"), TEXT("languageDetection"), diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp index d5a7742..12c5212 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp @@ -317,6 +317,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal TEXT("transcribe"), TEXT("vadSegments"), TEXT("grammarHints"), + TEXT("microphoneCaptureShell"), + TEXT("streamingPreview"), + TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), TEXT("wordTimestamps"), TEXT("languageDetection"), @@ -353,6 +356,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal TEXT("transcribe"), TEXT("vadSegments"), TEXT("grammarHints"), + TEXT("microphoneCaptureShell"), + TEXT("streamingPreview"), + TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), TEXT("wordTimestamps"), TEXT("languageDetection"), @@ -369,6 +375,10 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal { Health.ProviderLabel = ProviderLabel; } + Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell")); + Health.Capabilities.AddUnique(TEXT("streamingPreview")); + Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit")); + Health.SupportedOrchestrationProfiles.AddUnique(TEXT("python-batch-transcribe-v1")); if (Health.ServiceEndpoint.IsEmpty()) { Health.ServiceEndpoint = ServiceBaseUrl; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp index 4f60301..ea36744 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp @@ -128,6 +128,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHeal Health.ProviderLabel = TEXT("mock"); Health.ServiceEndpoint = TEXT("in-process://mock"); Health.Capabilities.AddUnique(TEXT("offline-transcribe")); + Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell")); + Health.Capabilities.AddUnique(TEXT("streamingPreview")); + Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit")); Health.Capabilities.AddUnique(TEXT("batchedTranscribe")); Health.Capabilities.AddUnique(TEXT("wordTimestamps")); Health.Capabilities.AddUnique(TEXT("languageDetection")); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp index 7ac5df7..9b2928f 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp @@ -2910,6 +2910,156 @@ namespace HyperTwistTrainingSubsystemInternal return BrowserShellState; } + FHyperTwistSpeechMicrophoneShellProfile BuildDefaultCoachCommandMicrophoneShellProfile(const FString& ProfileId) + { + FHyperTwistSpeechMicrophoneShellProfile Profile; + Profile.MicrophoneShellProfileId = ProfileId; + Profile.ShellKind = TEXT("microphone-overlay"); + Profile.CaptureMode = TEXT("vad-gated-short-command"); + Profile.StepWindowMs = 3000; + Profile.CaptureWindowMs = 10000; + Profile.KeepWindowMs = 200; + Profile.AudioContextTokens = 0; + Profile.VadThreshold = 0.6f; + Profile.HighPassFrequencyHz = 100.0f; + Profile.bKeepContextBetweenChunks = false; + Profile.bSupportsStreamingPreview = true; + Profile.bSupportsManualCommit = true; + + auto AddPanel = [&Profile]( + const TCHAR* PanelId, + const TCHAR* PanelKind, + const TCHAR* AnchorId, + const bool bVisibleByDefault + ) + { + FHyperTwistSpeechShellPanelLayout Panel; + Panel.PanelId = PanelId; + Panel.PanelKind = PanelKind; + Panel.AnchorId = AnchorId; + Panel.bVisibleByDefault = bVisibleByDefault; + Profile.Panels.Add(Panel); + }; + + auto AddAction = [&Profile]( + const TCHAR* ActionId, + const TCHAR* InputBinding, + const TCHAR* SurfaceId + ) + { + FHyperTwistSpeechShellActionBinding ActionBinding; + ActionBinding.ActionId = ActionId; + ActionBinding.InputBinding = InputBinding; + ActionBinding.SurfaceId = SurfaceId; + Profile.ActionBindings.Add(ActionBinding); + }; + + AddPanel(TEXT("microphone-status-readout"), TEXT("status-readout"), TEXT("top-left"), true); + AddPanel(TEXT("vad-meter"), TEXT("level-meter"), TEXT("left-stack-below-status"), true); + AddPanel(TEXT("transcript-preview"), TEXT("transcript-preview"), TEXT("bottom-left"), true); + AddPanel(TEXT("command-hints"), TEXT("command-hints"), TEXT("top-right"), true); + + AddAction(TEXT("pause-microphone-capture"), TEXT("P"), TEXT("microphone-status-readout")); + AddAction(TEXT("resume-microphone-capture"), TEXT("R"), TEXT("microphone-status-readout")); + AddAction(TEXT("commit-transcript"), TEXT("Enter"), TEXT("transcript-preview")); + AddAction(TEXT("close-speech-session"), TEXT("Escape"), TEXT("session-root")); + return Profile; + } + + FHyperTwistSpeechMicrophoneShellState BuildCoachCommandMicrophoneShellState( + const FHyperTwistSpeechSessionConfig& SessionConfig, + const FHyperTwistTrainingCompanionSpeechSessionState* PriorState, + const bool bSessionOpen + ) + { + const FHyperTwistSpeechMicrophoneShellProfile Profile = + SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid() + ? SessionConfig.MicrophoneShellProfileDefinition + : BuildDefaultCoachCommandMicrophoneShellProfile( + !SessionConfig.MicrophoneShellProfileId.IsEmpty() + ? SessionConfig.MicrophoneShellProfileId + : TEXT("coach-command-microphone-shell-v1") + ); + + FHyperTwistSpeechMicrophoneShellState ShellState; + ShellState.MicrophoneShellProfileId = Profile.MicrophoneShellProfileId; + ShellState.CaptureMode = Profile.CaptureMode; + ShellState.ListeningContractId = !SessionConfig.ListeningContractId.IsEmpty() + ? SessionConfig.ListeningContractId + : TEXT("listening-threshold-lifecycle"); + ShellState.InputRouteId = !SessionConfig.InputRouteId.IsEmpty() + ? SessionConfig.InputRouteId + : TEXT("queue/listening"); + ShellState.LastLanguageCode = + SessionConfig.LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase) + ? TEXT("en") + : SessionConfig.LanguageMode; + ShellState.StepWindowMs = Profile.StepWindowMs; + ShellState.CaptureWindowMs = Profile.CaptureWindowMs; + ShellState.KeepWindowMs = Profile.KeepWindowMs; + ShellState.VadThreshold = Profile.VadThreshold; + ShellState.bSessionOpen = bSessionOpen; + ShellState.bMicrophonePermissionGranted = true; + ShellState.bCaptureRouteReady = true; + ShellState.bCaptureActive = bSessionOpen; + ShellState.bVadArmed = SessionConfig.bEnableVad; + ShellState.bAwaitingSpeech = bSessionOpen; + ShellState.bTranscriptionInFlight = false; + ShellState.bCommitReady = false; + ShellState.bPermissionPromptVisible = false; + + for (const FHyperTwistSpeechShellPanelLayout& Panel : Profile.Panels) + { + if (Panel.PanelId == TEXT("microphone-status-readout")) + { + ShellState.bStatusPanelVisible = Panel.bVisibleByDefault; + } + else if (Panel.PanelId == TEXT("vad-meter")) + { + ShellState.bVadMeterVisible = Panel.bVisibleByDefault; + } + else if (Panel.PanelId == TEXT("transcript-preview")) + { + ShellState.bTranscriptPreviewVisible = Panel.bVisibleByDefault; + } + else if (Panel.PanelId == TEXT("command-hints")) + { + ShellState.bCommandHintsVisible = Panel.bVisibleByDefault; + } + } + + for (const FHyperTwistSpeechShellActionBinding& ActionBinding : Profile.ActionBindings) + { + ShellState.AvailableActionIds.Add(ActionBinding.ActionId); + } + + if (PriorState != nullptr) + { + ShellState.SubmittedUtteranceCount = FMath::Max(0, PriorState->SubmittedUtteranceCount); + ShellState.FinalTranscriptCount = FMath::Max(0, PriorState->FinalTranscriptCount); + ShellState.bCommitReady = + PriorState->bHasTranscriptResult && PriorState->LastTranscriptResult.IsStructurallyValid(); + ShellState.LastCommittedUtteranceId = + !PriorState->LastTranscriptResult.UtteranceId.IsEmpty() + ? PriorState->LastTranscriptResult.UtteranceId + : FString(); + ShellState.LastTranscriptPreview = PriorState->LastTranscriptResult.TranscriptText; + if (!PriorState->LastTranscriptResult.LanguageCode.IsEmpty()) + { + ShellState.LastLanguageCode = PriorState->LastTranscriptResult.LanguageCode; + } + + if (PriorState->MicrophoneShellState.IsStructurallyValid()) + { + ShellState.DetectedSpeechStartMs = PriorState->MicrophoneShellState.DetectedSpeechStartMs; + ShellState.DetectedSpeechEndMs = PriorState->MicrophoneShellState.DetectedSpeechEndMs; + ShellState.LastSilenceGapMs = PriorState->MicrophoneShellState.LastSilenceGapMs; + } + } + + return ShellState; + } + FHyperTwistVisionCorrectionState BuildClassicCubeCorrectionState( const FHyperTwistVisionSessionConfig& SessionConfig, const FHyperTwistVisionReconstructionSession& ReconstructionSession, @@ -8260,7 +8410,15 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out if (ActiveCompanionSpeechSessionState.bSessionOpen && ActiveCompanionSpeechSessionState.ActiveSessionId == SessionConfig.SessionId) { + const FHyperTwistTrainingCompanionSpeechSessionState PriorSessionState = + ActiveCompanionSpeechSessionState; ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig; + ActiveCompanionSpeechSessionState.MicrophoneShellState = + HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState( + SessionConfig, + &PriorSessionState, + true + ); ActiveCompanionSpeechSessionState.LastError.Reset(); ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset(); return true; @@ -8292,6 +8450,12 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig; ActiveCompanionSpeechSessionState.bHasTranscriptResult = false; ActiveCompanionSpeechSessionState.LastTranscriptResult = FHyperTwistSpeechTranscriptResult(); + ActiveCompanionSpeechSessionState.MicrophoneShellState = + HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState( + SessionConfig, + &ActiveCompanionSpeechSessionState, + true + ); RefreshCompanionSpeechServiceHealth(); return true; } @@ -8361,6 +8525,30 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp NormalizedUtterance.SpeechStartMs, NormalizedUtterance.SpeechEndMs ); + if (!ActiveCompanionSpeechSessionState.MicrophoneShellState.IsStructurallyValid()) + { + ActiveCompanionSpeechSessionState.MicrophoneShellState = + HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState( + ActiveCompanionSpeechSessionState.SessionConfig, + &ActiveCompanionSpeechSessionState, + true + ); + } + + ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId = NormalizedUtterance.UtteranceId; + ActiveCompanionSpeechSessionState.MicrophoneShellState.DetectedSpeechStartMs = + NormalizedUtterance.SpeechStartMs; + ActiveCompanionSpeechSessionState.MicrophoneShellState.DetectedSpeechEndMs = + NormalizedUtterance.SpeechEndMs; + ActiveCompanionSpeechSessionState.MicrophoneShellState.LastSilenceGapMs = + NormalizedUtterance.SilenceGapMs; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bSessionOpen = true; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = false; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bVadArmed = + ActiveCompanionSpeechSessionState.SessionConfig.bEnableVad; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = false; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = true; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bCommitReady = false; Result = SpeechClient->TranscribeSpeechUtterance(NormalizedUtterance); if (Result.SessionId.IsEmpty()) @@ -8449,6 +8637,28 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp ActiveCompanionSpeechSessionState.LastError = Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString(); ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError; + ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId.Reset(); + if (Result.bIsFinal && Result.IsStructurallyValid()) + { + ActiveCompanionSpeechSessionState.MicrophoneShellState.LastCommittedUtteranceId = Result.UtteranceId; + } + ActiveCompanionSpeechSessionState.MicrophoneShellState.LastTranscriptPreview = Result.TranscriptText; + if (!Result.LanguageCode.IsEmpty()) + { + ActiveCompanionSpeechSessionState.MicrophoneShellState.LastLanguageCode = Result.LanguageCode; + } + ActiveCompanionSpeechSessionState.MicrophoneShellState.SubmittedUtteranceCount = + ActiveCompanionSpeechSessionState.SubmittedUtteranceCount; + ActiveCompanionSpeechSessionState.MicrophoneShellState.FinalTranscriptCount = + ActiveCompanionSpeechSessionState.FinalTranscriptCount; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bSessionOpen = true; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = true; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bVadArmed = + ActiveCompanionSpeechSessionState.SessionConfig.bEnableVad; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = true; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = false; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bCommitReady = + Result.bIsFinal && Result.IsStructurallyValid(); return Result; } @@ -8487,6 +8697,16 @@ bool UHyperTwistTrainingSubsystem::CloseActiveCompanionSpeechSession(FString& Ou ActiveCompanionSpeechSessionState.ActiveSessionId.Reset(); ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601(); ActiveCompanionSpeechSessionState.LastError.Reset(); + ActiveCompanionSpeechSessionState.MicrophoneShellState = + HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState( + ActiveCompanionSpeechSessionState.SessionConfig, + &ActiveCompanionSpeechSessionState, + false + ); + ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId.Reset(); + ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = false; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = false; + ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = false; RefreshCompanionSpeechServiceHealth(); return true; } @@ -8989,6 +9209,28 @@ FHyperTwistSpeechSessionConfig UHyperTwistTrainingSubsystem::BuildActiveCompanio { SessionConfig.GrammarProfileId = TEXT("coach-command-grammar-v1"); } + if (SessionConfig.MicrophoneShellProfileId.IsEmpty() + && SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid()) + { + SessionConfig.MicrophoneShellProfileId = + SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId; + } + if (SessionConfig.MicrophoneShellProfileId.IsEmpty()) + { + SessionConfig.MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1"); + } + if (!SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid()) + { + SessionConfig.MicrophoneShellProfileDefinition = + HyperTwistTrainingSubsystemInternal::BuildDefaultCoachCommandMicrophoneShellProfile( + SessionConfig.MicrophoneShellProfileId + ); + } + if (SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId.IsEmpty()) + { + SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId = + SessionConfig.MicrophoneShellProfileId; + } SessionConfig.bEnableVad = true; if (!SessionConfig.VadPolicy.IsStructurallyValid()) { diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h index d472f7e..0d61bee 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h @@ -1434,6 +1434,270 @@ struct FHyperTwistSpeechTranscriptionOrchestrationProfile } }; +USTRUCT(BlueprintType) +struct FHyperTwistSpeechShellPanelLayout +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString PanelId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString PanelKind; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString AnchorId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bVisibleByDefault = true; + + bool IsStructurallyValid() const + { + return !PanelId.IsEmpty() + && !PanelKind.IsEmpty() + && !AnchorId.IsEmpty(); + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechShellActionBinding +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ActionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString InputBinding; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString SurfaceId; + + bool IsStructurallyValid() const + { + return !ActionId.IsEmpty() + && !InputBinding.IsEmpty() + && !SurfaceId.IsEmpty(); + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechMicrophoneShellProfile +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString MicrophoneShellProfileId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ShellKind = TEXT("microphone-overlay"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CaptureMode = TEXT("vad-gated-short-command"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 StepWindowMs = 3000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 CaptureWindowMs = 10000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 KeepWindowMs = 200; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 AudioContextTokens = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float VadThreshold = 0.6f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float HighPassFrequencyHz = 100.0f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bKeepContextBetweenChunks = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSupportsStreamingPreview = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSupportsManualCommit = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Panels; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray ActionBindings; + + bool IsStructurallyValid() const + { + if (MicrophoneShellProfileId.IsEmpty() + || ShellKind.IsEmpty() + || CaptureMode.IsEmpty() + || StepWindowMs <= 0 + || CaptureWindowMs < StepWindowMs + || KeepWindowMs < 0 + || KeepWindowMs > CaptureWindowMs + || AudioContextTokens < 0 + || VadThreshold <= 0.0f + || HighPassFrequencyHz < 0.0f + || Panels.Num() <= 0 + || ActionBindings.Num() <= 0) + { + return false; + } + + for (const FHyperTwistSpeechShellPanelLayout& Panel : Panels) + { + if (!Panel.IsStructurallyValid()) + { + return false; + } + } + + for (const FHyperTwistSpeechShellActionBinding& ActionBinding : ActionBindings) + { + if (!ActionBinding.IsStructurallyValid()) + { + return false; + } + } + + return true; + } +}; + +USTRUCT(BlueprintType) +struct FHyperTwistSpeechMicrophoneShellState +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString MicrophoneShellProfileId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CaptureMode = TEXT("vad-gated-short-command"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ListeningContractId = TEXT("listening-threshold-lifecycle"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString InputRouteId = TEXT("queue/listening"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString ActiveUtteranceId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastCommittedUtteranceId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastTranscriptPreview; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastLanguageCode = TEXT("en"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 StepWindowMs = 3000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 CaptureWindowMs = 10000; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 KeepWindowMs = 200; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 SubmittedUtteranceCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 FinalTranscriptCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 DetectedSpeechStartMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 DetectedSpeechEndMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 LastSilenceGapMs = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + float VadThreshold = 0.6f; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSessionOpen = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bMicrophonePermissionGranted = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bCaptureRouteReady = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bCaptureActive = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bVadArmed = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bAwaitingSpeech = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bTranscriptionInFlight = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bCommitReady = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bStatusPanelVisible = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bVadMeterVisible = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bTranscriptPreviewVisible = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bCommandHintsVisible = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bPermissionPromptVisible = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray AvailableActionIds; + + bool IsStructurallyValid() const + { + if (MicrophoneShellProfileId.IsEmpty() + || CaptureMode.IsEmpty() + || ListeningContractId.IsEmpty() + || InputRouteId.IsEmpty() + || LastLanguageCode.IsEmpty() + || StepWindowMs <= 0 + || CaptureWindowMs < StepWindowMs + || KeepWindowMs < 0 + || KeepWindowMs > CaptureWindowMs + || SubmittedUtteranceCount < 0 + || FinalTranscriptCount < 0 + || DetectedSpeechStartMs < 0 + || DetectedSpeechEndMs < DetectedSpeechStartMs + || LastSilenceGapMs < 0 + || VadThreshold <= 0.0f + || AvailableActionIds.Num() <= 0) + { + return false; + } + + for (const FString& ActionId : AvailableActionIds) + { + if (ActionId.IsEmpty()) + { + return false; + } + } + + return true; + } +}; + USTRUCT(BlueprintType) struct FHyperTwistSpeechSessionConfig { @@ -1466,6 +1730,12 @@ struct FHyperTwistSpeechSessionConfig UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FString GrammarProfileId = TEXT("coach-command-grammar-v1"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechMicrophoneShellProfile MicrophoneShellProfileDefinition; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") bool bEnableVad = true; @@ -1490,6 +1760,8 @@ struct FHyperTwistSpeechSessionConfig && SampleRateHz > 0 && ChannelCount > 0 && !TaskKind.IsEmpty() + && !MicrophoneShellProfileId.IsEmpty() + && MicrophoneShellProfileDefinition.IsStructurallyValid() && VadPolicy.IsStructurallyValid() && OrchestrationProfile.IsStructurallyValid(); } diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h index bd71e81..88842e1 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h @@ -1937,6 +1937,9 @@ struct FHyperTwistTrainingCompanionSpeechSessionState UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FHyperTwistSpeechServiceHealth ServiceHealth; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechMicrophoneShellState MicrophoneShellState; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FHyperTwistSpeechTranscriptResult LastTranscriptResult; }; diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RUMicrophoneShellContractTest.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RUMicrophoneShellContractTest.cpp new file mode 100644 index 0000000..9317cf7 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RUMicrophoneShellContractTest.cpp @@ -0,0 +1,278 @@ +// Copyright HyperTwist, Inc. All Rights Reserved. + +#include "Misc/AutomationTest.h" + +#include "Engine/GameInstance.h" +#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h" +#include "UObject/UnrealType.h" + +#if WITH_AUTOMATION_TESTS + +namespace HyperTwistWhisperCppPhase6RUTestInternal +{ + FHyperTwistTrainingDeck MakeSpeechDeck() + { + FHyperTwistTrainingDeck Deck; + Deck.DeckId = TEXT("phase6r-u/whispercpp-microphone-shell"); + Deck.Title = TEXT("Phase 6R-U Whisper Microphone Shell"); + Deck.DeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + + FHyperTwistTrainingCase TrainingCase; + TrainingCase.CaseId = TEXT("phase6r-u-case"); + TrainingCase.PuzzleId = TEXT("cube/3x3x3"); + TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence; + TrainingCase.PromptLabel = TEXT("Phase 6R-U Coach Speech"); + TrainingCase.AllowedDeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + Deck.Cases = {TrainingCase}; + return Deck; + } + + void ForceMockSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem) + { + if (TrainingSubsystem == nullptr) + { + return; + } + + if (FStrProperty* SpeechClientKindProperty = FindFProperty( + UHyperTwistTrainingSubsystem::StaticClass(), + TEXT("CompanionSpeechClientKind") + )) + { + SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, TEXT("mock")); + } + } + + UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId) + { + UGameInstance* GameInstance = NewObject(GetTransientPackage()); + if (GameInstance == nullptr) + { + return nullptr; + } + + UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject(GameInstance); + if (TrainingSubsystem == nullptr) + { + return nullptr; + } + + ForceMockSpeechClient(TrainingSubsystem); + const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck( + MakeSpeechDeck(), + TEXT("phase6r-u-user"), + SessionId, + EHyperTwistTrainingDeliveryMode::CoachReviewed + ); + return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr; + } + + const FHyperTwistSpeechShellPanelLayout* FindPanel( + const FHyperTwistSpeechMicrophoneShellProfile& Profile, + const FString& PanelId + ) + { + return Profile.Panels.FindByPredicate( + [&PanelId](const FHyperTwistSpeechShellPanelLayout& Candidate) + { + return Candidate.PanelId == PanelId; + } + ); + } + + const FHyperTwistSpeechShellActionBinding* FindAction( + const FHyperTwistSpeechMicrophoneShellProfile& Profile, + const FString& ActionId + ) + { + return Profile.ActionBindings.FindByPredicate( + [&ActionId](const FHyperTwistSpeechShellActionBinding& Candidate) + { + return Candidate.ActionId == ActionId; + } + ); + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RUMicrophoneShellSessionConfigTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellSessionConfig", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RUMicrophoneShellSessionConfigTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-config-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue(TEXT("The microphone shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellProfile& Profile = + SessionState.SessionConfig.MicrophoneShellProfileDefinition; + + TestEqual( + TEXT("The microphone shell route must preserve the first-party shell profile id."), + SessionState.SessionConfig.MicrophoneShellProfileId, + TEXT("coach-command-microphone-shell-v1") + ); + TestTrue(TEXT("The microphone shell route must expose a structurally valid shell profile."), Profile.IsStructurallyValid()); + TestEqual(TEXT("The microphone shell must retain four bounded panels."), Profile.Panels.Num(), 4); + TestEqual(TEXT("The microphone shell must retain four bounded actions."), Profile.ActionBindings.Num(), 4); + TestEqual(TEXT("The microphone shell must preserve the donor-backed step window."), Profile.StepWindowMs, 3000); + TestEqual(TEXT("The microphone shell must preserve the donor-backed capture window."), Profile.CaptureWindowMs, 10000); + TestEqual(TEXT("The microphone shell must preserve the donor-backed keep window."), Profile.KeepWindowMs, 200); + TestEqual(TEXT("The microphone shell must preserve zero audio-context tokens in the bounded route."), Profile.AudioContextTokens, 0); + TestTrue(TEXT("The microphone shell must preserve the donor-backed VAD threshold."), FMath::IsNearlyEqual(Profile.VadThreshold, 0.6f)); + TestTrue(TEXT("The microphone shell must preserve the donor-backed high-pass floor."), FMath::IsNearlyEqual(Profile.HighPassFrequencyHz, 100.0f)); + TestFalse(TEXT("The microphone shell must keep previous-text carryover disabled in the bounded route."), Profile.bKeepContextBetweenChunks); + TestTrue(TEXT("The microphone shell must advertise streaming preview."), Profile.bSupportsStreamingPreview); + TestTrue(TEXT("The microphone shell must advertise manual commit."), Profile.bSupportsManualCommit); + + const FHyperTwistSpeechShellPanelLayout* VadPanel = + HyperTwistWhisperCppPhase6RUTestInternal::FindPanel(Profile, TEXT("vad-meter")); + TestNotNull(TEXT("The VAD meter panel must be present."), VadPanel); + if (VadPanel != nullptr) + { + TestEqual(TEXT("The VAD meter must stay anchored below the status readout."), VadPanel->AnchorId, TEXT("left-stack-below-status")); + } + + const FHyperTwistSpeechShellActionBinding* CommitAction = + HyperTwistWhisperCppPhase6RUTestInternal::FindAction(Profile, TEXT("commit-transcript")); + TestNotNull(TEXT("The manual transcript commit action must be present."), CommitAction); + if (CommitAction != nullptr) + { + TestEqual(TEXT("The manual transcript commit action must keep the Enter binding."), CommitAction->InputBinding, TEXT("Enter")); + } + + TestTrue( + TEXT("The speech service health must expose microphone-shell capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("microphoneCaptureShell")) + ); + TestTrue( + TEXT("The speech service health must expose streaming-preview capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("streamingPreview")) + ); + TestTrue( + TEXT("The speech service health must expose manual transcript commit capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("manualTranscriptCommit")) + ); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RUMicrophoneShellOpenStateTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellOpenState", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RUMicrophoneShellOpenStateTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-open-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue(TEXT("The microphone shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState; + + TestTrue(TEXT("The microphone shell open state must be structurally valid."), ShellState.IsStructurallyValid()); + TestEqual(TEXT("The microphone shell state must preserve the first-party profile id."), ShellState.MicrophoneShellProfileId, TEXT("coach-command-microphone-shell-v1")); + TestEqual(TEXT("The microphone shell state must preserve the listening contract."), ShellState.ListeningContractId, TEXT("listening-threshold-lifecycle")); + TestEqual(TEXT("The microphone shell state must preserve the input route."), ShellState.InputRouteId, TEXT("queue/listening")); + TestEqual(TEXT("The microphone shell state must preserve the donor-backed step window."), ShellState.StepWindowMs, 3000); + TestEqual(TEXT("The microphone shell state must preserve the donor-backed capture window."), ShellState.CaptureWindowMs, 10000); + TestEqual(TEXT("The microphone shell state must preserve the donor-backed keep window."), ShellState.KeepWindowMs, 200); + TestTrue(TEXT("The microphone shell must report the session open."), ShellState.bSessionOpen); + TestTrue(TEXT("The microphone shell must report the capture route ready."), ShellState.bCaptureRouteReady); + TestTrue(TEXT("The microphone shell must start capture-active."), ShellState.bCaptureActive); + TestTrue(TEXT("The microphone shell must start with VAD armed."), ShellState.bVadArmed); + TestTrue(TEXT("The microphone shell must start awaiting speech."), ShellState.bAwaitingSpeech); + TestFalse(TEXT("The microphone shell must not start with transcription in flight."), ShellState.bTranscriptionInFlight); + TestFalse(TEXT("The microphone shell must not start commit-ready."), ShellState.bCommitReady); + TestTrue(TEXT("The microphone shell must keep the status panel visible."), ShellState.bStatusPanelVisible); + TestTrue(TEXT("The microphone shell must keep the VAD meter visible."), ShellState.bVadMeterVisible); + TestTrue(TEXT("The microphone shell must keep the transcript preview visible."), ShellState.bTranscriptPreviewVisible); + TestTrue(TEXT("The microphone shell must keep the command hints visible."), ShellState.bCommandHintsVisible); + TestFalse(TEXT("The microphone shell must keep the permission prompt hidden."), ShellState.bPermissionPromptVisible); + TestEqual(TEXT("The microphone shell must start with no submitted utterances."), ShellState.SubmittedUtteranceCount, 0); + TestEqual(TEXT("The microphone shell must start with no final transcripts."), ShellState.FinalTranscriptCount, 0); + TestEqual(TEXT("The microphone shell must expose four bounded shell actions."), ShellState.AvailableActionIds.Num(), 4); + TestTrue(TEXT("The microphone shell must expose pause capture."), ShellState.AvailableActionIds.Contains(TEXT("pause-microphone-capture"))); + TestTrue(TEXT("The microphone shell must expose resume capture."), ShellState.AvailableActionIds.Contains(TEXT("resume-microphone-capture"))); + TestTrue(TEXT("The microphone shell must expose manual transcript commit."), ShellState.AvailableActionIds.Contains(TEXT("commit-transcript"))); + TestTrue(TEXT("The microphone shell must expose session close."), ShellState.AvailableActionIds.Contains(TEXT("close-speech-session"))); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RUMicrophoneShellUtteranceStateTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellUtteranceState", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RUMicrophoneShellUtteranceStateTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-utterance-session")); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FHyperTwistSpeechUtteranceEnvelope Utterance; + Utterance.AudioRef = TEXT("mock://coach/next-case"); + Utterance.SpeechStartMs = 140; + Utterance.SpeechEndMs = 980; + Utterance.SilenceGapMs = 240; + Utterance.SampleCount = 13440; + const FHyperTwistSpeechTranscriptResult Result = + TrainingSubsystem->SubmitActiveCompanionSpeechUtterance(Utterance); + + TestTrue(TEXT("The microphone shell utterance route must return a structurally valid transcript result."), Result.IsStructurallyValid()); + TestEqual(TEXT("The bounded mock route must resolve the next-case transcript."), Result.TranscriptText, TEXT("next case")); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState; + + TestTrue(TEXT("The microphone shell utterance state must be structurally valid."), ShellState.IsStructurallyValid()); + TestTrue(TEXT("The microphone shell must keep the speech session open after a final utterance."), ShellState.bSessionOpen); + TestTrue(TEXT("The microphone shell must resume capture after a final utterance."), ShellState.bCaptureActive); + TestTrue(TEXT("The microphone shell must resume awaiting speech after a final utterance."), ShellState.bAwaitingSpeech); + TestFalse(TEXT("The microphone shell must clear transcription-in-flight after a final utterance."), ShellState.bTranscriptionInFlight); + TestTrue(TEXT("The microphone shell must become commit-ready after a final utterance."), ShellState.bCommitReady); + TestTrue(TEXT("The microphone shell must clear the active utterance id after a final utterance."), ShellState.ActiveUtteranceId.IsEmpty()); + TestEqual(TEXT("The microphone shell must retain the last committed utterance id."), ShellState.LastCommittedUtteranceId, Result.UtteranceId); + TestEqual(TEXT("The microphone shell must retain the last transcript preview."), ShellState.LastTranscriptPreview, TEXT("next case")); + TestEqual(TEXT("The microphone shell must retain the transcript language."), ShellState.LastLanguageCode, TEXT("en")); + TestEqual(TEXT("The microphone shell must count one submitted utterance."), ShellState.SubmittedUtteranceCount, 1); + TestEqual(TEXT("The microphone shell must count one final transcript."), ShellState.FinalTranscriptCount, 1); + TestEqual(TEXT("The microphone shell must retain the detected speech start."), ShellState.DetectedSpeechStartMs, 140); + TestEqual(TEXT("The microphone shell must retain the detected speech end."), ShellState.DetectedSpeechEndMs, 980); + TestEqual(TEXT("The microphone shell must retain the last silence gap."), ShellState.LastSilenceGapMs, 240); + return true; +} + +#endif diff --git a/docs/REPO_LICENSE_TRACKING.md b/docs/REPO_LICENSE_TRACKING.md index c29b35e..e58a0e4 100644 --- a/docs/REPO_LICENSE_TRACKING.md +++ b/docs/REPO_LICENSE_TRACKING.md @@ -167,9 +167,13 @@ Status update on `2026-05-21`: control pass is now consumed - the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping packet is now landed in current code -- the current next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp` - live microphone capture shell preparation/control pass, not a new restrictive packet by - default +- the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell + control pass is now consumed +- the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell + packet is now landed in current code +- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` + device-permission and capture-route readiness shell preparation/control pass, not a new + restrictive packet by default - use the repo-row README census, the portfolio standing refresh backfill, and the `2R-A` ownership contract for the current queue after that correction @@ -273,8 +277,8 @@ Current practical interpretation: - the landed `PostHog/posthog` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_D_POSTHOG_CONTROL_PLANE_IMPLEMENTATION_2026-05-13.md` - the landed `screenpipe/screenpipe` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_E_SCREENPIPE_CAPTURE_HISTORY_IMPLEMENTATION_2026-05-13.md` - the landed `remotion-dev/remotion` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_F_REMOTION_MEDIA_EXPORT_IMPLEMENTATION_2026-05-13.md` -- the next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp` - live microphone capture shell preparation/control pass +- the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` + device-permission and capture-route readiness shell preparation/control pass Companion docs: @@ -1522,9 +1526,12 @@ Approved working posture: - keep it bounded behind a speech-input seam - prefer command, dictation, and constrained coach-interaction use over broad “voice assistant platform” scope - the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle +- the next bounded permissive implementation slice is now landed as a live microphone shell + profile and shell-state boundary above the existing transcript-session seam - the top-level provider/session contract remains first-party HyperTwist-owned and provider-neutral; this row does not own that lane -- keep model files, downloadable payloads, and future voice-asset review separate from the code-license judgment +- keep real device-permission workflow, model files, downloadable payloads, and future + voice-asset review separate from the code-license judgment - treat `SYSTRAN/faster-whisper` as the landed complementary Python-orchestration donor rather than widening `whisper.cpp` into the runtime core diff --git a/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md b/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md new file mode 100644 index 0000000..e58c433 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md @@ -0,0 +1,160 @@ +# HyperTwist Phase 6R-U whisper.cpp live microphone capture shell implementation packet + +Created on `2026-05-24` + +## Status + +- first-party HyperTwist packet +- bounded permissive `Phase 6R-U` implementation slice + +## Purpose + +This packet lands the next narrower bounded slice from the retained +`ggml-org/whisper.cpp` speech-input row. + +The landed slice is: + +- first-party live microphone capture shell profile and session-state boundary + above the existing companion speech transcript-session seam + +It is not: + +- a full `whisper.cpp` row transplant +- a real device-permission workflow packet +- a downloadable model or payload-shipping packet +- a top-level provider/session contract packet +- a voice-output packet +- a broad assistant-platform packet + +## Current authority basis + +This implementation packet stands on: + +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/ops/HYPERTWIST_CROSS_LANE_AUTHORITY_HIERARCHY_AND_RECONCILIATION_2026-05-20.md` +- `docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_PREPARATION_PACKET_2026-05-24.md` + +The retained owner remains: + +- `ggml-org/whisper.cpp` + +The top-level provider/session contract owner does not change here: + +- first-party HyperTwist remains the provider-neutral owner for the speech + provider/session lane +- `SYSTRAN/faster-whisper` remains the complementary landed Python + orchestration donor +- this packet keeps `ggml-org/whisper.cpp` bounded to the narrower live + microphone shell donor slice only + +The granted family remains bounded to: + +- shell panel and action contract shaping +- donor-backed microphone shell defaults +- session open / submit / close shell-state composition +- speech-health capability exposure for the bounded shell + +This packet lands only the first narrower family in that granted set. + +## Landed scope + +The current code now owns a retained live microphone shell contract through: + +- retained recognition contract types for: + - `FHyperTwistSpeechShellPanelLayout` + - `FHyperTwistSpeechShellActionBinding` + - `FHyperTwistSpeechMicrophoneShellProfile` + - `FHyperTwistSpeechMicrophoneShellState` + - microphone-shell-aware `FHyperTwistSpeechSessionConfig` +- companion speech session state expansion in: + - `FHyperTwistTrainingCompanionSpeechSessionState` +- sample session-config and service-health outputs in: + - `UHyperTwistContractLibrary` +- direct-donor speech client seam capability exposure in: + - `UHyperTwistMockSpeechClient` + - `UHyperTwistHttpSpeechClient` +- active companion shell-state normalization in: + - `UHyperTwistTrainingSubsystem` +- focused automation coverage in: + - `HyperTwistWhisperCppPhase6RUMicrophoneShellContractTest.cpp` + +## Why this is still intentionally bounded + +This packet lands the next donor-backed microphone shell seam, but it does not +widen into the neighboring retained families. + +Still deferred: + +- real device-permission workflow +- native audio-device route ownership beyond bounded shell state +- downloadable model or payload shipping +- provider-profile and BYOK custody +- voice-output ownership +- broad assistant-platform scope + +## Validation + +Build validation: + +- `C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE -NoUba` + +Focused automation validation: + +- `C:\Program Files\Epic Games\UE_5.7\Engine\Binaries\Win64\UnrealEditor-Cmd.exe C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject -unattended -nop4 -nosplash -NullRHI -log -stdout -FullStdOutLogOutput -AbsLog=C:\HyperTwist\UnrealHyperTwist\Saved\Logs\Phase6R-U-WhisperCpp-Verify.log -ReportExportPath=C:\HyperTwist\UnrealHyperTwist\Saved\AutomationReports\Phase6R-U-WhisperCpp-Verify -ExecCmds="Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.U; Quit" -TestExit="Automation Test Queue Empty"` + +Regression automation validation: + +- `HyperTwist.Permissive.WhisperCpp.Phase6R.E` +- `HyperTwist.Permissive.FasterWhisper.Phase6R.F` +- `HyperTwist.Permissive.Piper.Phase6R.G` +- `HyperTwist.Permissive.Coqui.Phase6R.H` +- `HyperTwist.CleanRoom.CubeDesk` + +Expected covered tests: + +- `MicrophoneShellSessionConfig` +- `MicrophoneShellOpenState` +- `MicrophoneShellUtteranceState` +- existing `Phase 6R-E`, `Phase 6R-F`, `Phase 6R-G`, `Phase 6R-H`, and + `CubeDesk` regression suites + +## Queue effect + +This packet consumes the current `Phase 6R-U` implementation slice. + +`ggml-org/whisper.cpp` remains only partially incorporated: + +- landed now: + - speech transcript session boundary + - VAD-aware session config and utterance envelope + - transcript segment and transcript result + - sidecar health and speech client boundary + - live microphone shell profile and shell-state boundary + - bounded shell capability exposure for: + - microphone capture shell + - streaming preview + - manual transcript commit +- still deferred: + - real device-permission workflow + - native capture-route readiness beyond bounded shell state + - downloadable model or payload shipping + - provider-profile and BYOK custody + - broad assistant-platform scope + +The next clean move is: + +- a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell preparation/control pass + +Keep the future sequencing guards visible: + +- `ggml-org/whisper.cpp` + - keep code-license judgments separate from downloadable model or payload + license review in any follow-on microphone packet +- first-party HyperTwist + - keep the top-level provider/session contract first-party rather than + letting `whisper.cpp` or `faster-whisper` absorb that lane through a + narrower donor win +- `SYSTRAN/faster-whisper` + - keep the row complementary to the landed shell and bounded to Python + orchestration/service-lane ownership rather than shell ownership diff --git a/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_PREPARATION_PACKET_2026-05-24.md b/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_PREPARATION_PACKET_2026-05-24.md new file mode 100644 index 0000000..3999c45 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_PREPARATION_PACKET_2026-05-24.md @@ -0,0 +1,179 @@ +# HyperTwist Phase 6R-U whisper.cpp live microphone capture shell preparation packet + +Created on `2026-05-24` + +## Status + +- historical same-day preparation authority +- bounded post-`Phase 6R-T` control slice +- the first bounded implementation slice now lands separately under: + - `docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md` + +## Purpose + +This packet freezes the next widening order after the landed `Phase 6R-T` +`MagicTile` transform-aware macro remapping slice. + +The open task was: + +- define the next bounded source-backed widening packet for + `ggml-org/whisper.cpp` as the retained `Phase 6R-U` live microphone capture + shell seam + +It is not: + +- a full `whisper.cpp` row transplant +- a real device-permission workflow packet +- a downloadable model or payload-shipping packet +- a `SYSTRAN/faster-whisper` provider/service rewrite packet +- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet +- a broad assistant-platform packet + +## Current authority basis + +This preparation packet stands on already-closed authority: + +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md` +- `docs/arch/HYPERTWIST_PHASE6R_T_MAGICTILE_TRANSFORM_AWARE_MACRO_REMAPPING_IMPLEMENTATION_PACKET_2026-05-24.md` + +The key accepted routing facts are: + +- `ggml-org/whisper.cpp` remains the retained offline STT donor after the + already landed `Phase 6R-E` transcript-session slice +- `SYSTRAN/faster-whisper` remains the complementary landed Python + orchestration donor above the existing first-party speech session boundary +- first-party HyperTwist already owns the top-level provider/session contract, + companion listening lifecycle, and speech-session substrate +- the retained donor value strongest for the next narrow widening is: + - VAD-gated streaming-shell defaults + - rolling capture-window and keep-window posture + - transcript-preview and manual-commit shell cues + - health-surface capability exposure for the bounded shell +- ownership denied here is: + - do not widen into real device-permission workflow ownership + - do not widen into downloadable model or payload shipping + - do not let `whisper.cpp` inherit the top-level provider/session lane + - do not widen into voice-output or broad assistant-platform ownership + +## Required result + +The source-backed control pass for this packet is now complete. + +The first actual `6R-U` implementation packet should: + +- define one bounded live microphone capture shell slice above the landed + transcript-session boundary +- land a first-party microphone shell profile and session-state surface +- preserve donor-backed stream defaults as shell metadata rather than a broader + runtime takeover +- add bounded health-surface capability exposure and focused automation +- explicitly state which neighboring retained families stay closed + +## Source-backed retained basis + +The queue-head decision is now source-backed rather than README-only. + +Inspected retained donor basis: + +- `C:\visual_studio_solutions\multi_project\GPT 5.4 HyperTwist parse\49-ggml-org-whisper-cpp-upstream-dossier.md` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\stream\stream.cpp` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\server\README.md` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\include\whisper.h` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\tests\test-vad.cpp` + +Inspected first-party receiving basis: + +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp` + +## Narrowed 6R-U slice decision + +The next widening slice is fixed as: + +1. live microphone capture shell profile and session-state boundary + +That narrowed slice covers: + +- first-party speech shell contract types for: + - shell panel layout + - shell action binding + - microphone shell profile + - microphone shell state +- donor-backed shell defaults for: + - `3000ms` step window + - `10000ms` capture window + - `200ms` keep window + - `0.6` VAD threshold + - `100Hz` high-pass threshold + - no previous-text carryover by default +- first-party session-state composition for: + - open-state shell activation + - utterance preview/commit progression + - transcript-preview, speech-gap, and last-utterance readout +- bounded mock and HTTP speech-health capability exposure for: + - microphone capture shell + - streaming preview + - manual transcript commit +- focused automation coverage + +## Deferred neighboring families + +The first `6R-U` implementation packet must keep these capability families +closed: + +- real device-permission workflow ownership +- native audio-device route ownership beyond bounded shell state +- downloadable model or payload shipping +- provider-profile or BYOK custody +- voice-output ownership +- broad assistant-platform scope + +Why this slice is next: + +- `whisper.cpp` exposes a narrower donor seam at VAD-gated streaming shell + posture than at broader model/runtime ownership +- the already landed `Phase 6R-E` and `Phase 6R-F` packets provide the right + local substrate for a bounded shell layer above transcript sessions and + orchestration metadata +- widening directly into permission workflow, model checkout, or payload + shipping would blur custody and legal boundaries more than clarify them at + this point + +## Acceptance criteria + +- the packet records why the queue advanced from landed `Phase 6R-T` to + `Phase 6R-U` +- the packet records that the first `6R-U` widening slice is live microphone + capture shell profile/state only +- the packet states exact out-of-scope families for the first `6R-U` pass +- the packet preserves the upstream `MIT` attribution boundary explicitly +- the packet leaves permission workflow, payload shipping, and broad + assistant-platform ownership deferred + +## Validation checklist + +1. confirm the landed `Phase 6R-T` packet is the consumed prior queue head +2. confirm the retained `whisper.cpp` seam narrows cleanly to shell profile and + shell-state composition +3. confirm the next implementation packet is framed as bounded source-backed + widening rather than a broad speech-platform transplant + +That is the packet. + +## Queue effect + +This preparation packet is now consumed by the landed bounded `Phase 6R-U` +implementation slice. + +Once that slice lands, the next clean move should stay narrow inside the +retained `whisper.cpp` row at: + +- a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell preparation/control pass diff --git a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md index f2a5661..d76ca37 100644 --- a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md +++ b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md @@ -116,14 +116,18 @@ The next bounded move is now: control pass is now consumed 14. the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping packet is now landed in current code -15. the next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp` - live microphone capture shell preparation/control pass -16. keep the speech-lane guard visible: +15. the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell + control pass is now consumed +16. the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell + packet is now landed in current code +17. the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` + device-permission and capture-route readiness shell preparation/control pass +18. keep the speech-lane guard visible: - keep code-license judgments separate from model, voice, and payload-license review -17. keep the provider-neutral speech-lane guard visible: +19. keep the provider-neutral speech-lane guard visible: - first-party HyperTwist owns the top-level provider/session contract - `whisper.cpp` and `faster-whisper` may win narrower donor slices without inheriting that lane -18. keep the `MagicTile` guard visible: +20. keep the `MagicTile` guard visible: - keep broad non-Euclidean interaction or WinForms/OpenTK host-shell ownership closed by default unless a narrower first-party gap is proven above the landed macro-remapping seam diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md index a7c11e3..4eea805 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md @@ -120,6 +120,7 @@ Do not collapse those three tiers into one undifferentiated "features" voice. | Transform-aware tiling macro remapping and persistence boundary | Implemented now | landed `MagicTile` `Phase 6R-T` | First-party transform-aware remapping, reverse/setup-move playback tags, and macro XML compatibility boundary are live above the landed topology slice. | | Provider-backed recognition session boundary | Implemented now | first-party current code | `UHyperTwistHttpVisionClient` and normalized recognition envelopes are live first-party seams. | | Speech transcript session boundary | Implemented now | landed `whisper.cpp` `Phase 6R-E` | Transcript sessions, utterance envelopes, and speech-health seams are live. | +| Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` | First-party microphone shell profile, bounded panel/action contract, and live shell-state composition are now live above the transcript-session seam. | | Analytics/reporting surfaces | Implemented now | landed analytics/reporting packets | Reporting is real, but bounded to accepted retained slices. | | Browser/spatial/media adjunct surfaces | Implemented now | landed `three.js`, `react-three-fiber`, `xr`, `model-viewer`, `remotion` packets | These are implemented bounded families, not proof of unlimited browser-shell parity. | @@ -197,12 +198,13 @@ repo. | Feature | Status | Primary authority | Notes | |---|---|---|---| | Speech transcript session boundary | Implemented now | landed `whisper.cpp` packet | Current bounded STT truth. | +| Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` packet | Current bounded microphone shell truth above the landed transcript-session and orchestration seams. | | Provider-backed speech session health and transcript envelopes | Implemented now | first-party speech client surfaces | Current first-party speech seam. | | Python transcription-service orchestration profile | Implemented now | landed `faster-whisper` packet | Current bounded Python STT orchestration truth above the existing speech session boundary. | | Local narration sidecar seam | Implemented now | landed `piper` packet | Current bounded offline narration contract, voice catalog, and local sidecar truth. | | Advanced narration orchestration profile | Implemented now | landed `coqui-ai/TTS` packet | Current bounded richer narration profile above the existing companion narration contract. | | Broad voice output / narration sidecars | Deep-source grounded retained | `piper`, `coqui-ai/TTS` retained remainder | Keep downloadable voice/model review separate from the code-license judgment. | -| Broad live microphone shell and downloadable payload shipping | Deep-source grounded retained | speech stack retained remainder | Explicitly deferred. | +| Device-permission shell and downloadable payload shipping | Deep-source grounded retained | `whisper.cpp` retained remainder | Keep permission workflow, native capture-route readiness, and downloadable model/payload review separate from the landed bounded microphone shell. | ### 8. Provider-neutral AI/provider routing diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md index fe63a48..6375386 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md @@ -87,15 +87,15 @@ Canonical discovery surfaces for roadmap interpretation: consumed - the bounded permissive `Phase 6R-E` `ggml-org/whisper.cpp` speech transcript session packet is now landed in current code -- `ggml-org/whisper.cpp` remains partially incorporated; live microphone capture, model/payload - shipping, Python orchestration, and voice-output ownership stay deferred +- `ggml-org/whisper.cpp` remains partially incorporated; live microphone shell is now landed, but + device-permission workflow, capture-route readiness, and model/payload shipping stay deferred - the later provider-family backfill keeps the top-level provider/session contract first-party and provider-neutral rather than donor-owned - the generic source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` control pass is now consumed - the bounded permissive `Phase 6R-F` `SYSTRAN/faster-whisper` transcription-service orchestration packet is now landed in current code -- `SYSTRAN/faster-whisper` remains partially incorporated; microphone capture, BYOK/profile - custody, payload shipping, and voice-output ownership stay deferred +- `SYSTRAN/faster-whisper` remains partially incorporated; richer live service-lane tuning, + BYOK/profile custody, payload shipping, and voice-output ownership stay deferred - the generic source-backed `Phase 6R-G` `rhasspy/piper` control pass is now consumed - the bounded permissive `Phase 6R-G` `rhasspy/piper` local narration sidecar packet is now landed in current code @@ -143,9 +143,13 @@ Canonical discovery surfaces for roadmap interpretation: control pass is now consumed - the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping packet is now landed in current code -- the current next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp` - live microphone capture shell preparation/control pass, while keeping speech model/payload - review separate from the code-license judgment +- the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell + control pass is now consumed +- the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell packet + is now landed in current code +- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` + device-permission and capture-route readiness shell preparation/control pass, while keeping + downloadable model/payload review separate from the code-license judgment - the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than waiting on another first-party packet @@ -207,14 +211,14 @@ Current routing truth: - active non-live implementation-board rows: `34` - retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9` -The next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp` -live microphone capture shell preparation/control pass. +The next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` +device-permission and capture-route readiness shell preparation/control pass. Queue interpretation after that control pass: - then continue with the retained repo-row queue - highest current repo-row queue pressure sits in: - - the retained `whisper.cpp` live microphone capture shell remainder + - the retained `whisper.cpp` device-permission and capture-route readiness shell remainder - `HactarCE/Hyperspeedcube` is now closed for the currently justified retained row: - landed: - puzzle catalog contract @@ -298,11 +302,15 @@ Queue interpretation after that control pass: - VAD-aware session config and utterance envelope - transcript segment and transcript result - sidecar health and speech client boundary + - live microphone shell profile and shell-state boundary + - bounded shell capability exposure for: + - microphone capture shell + - streaming preview + - manual transcript commit - still deferred: - - live microphone capture shell + - real device-permission workflow + - native capture-route readiness beyond bounded shell state - downloadable model or payload shipping - - Python orchestration / batching / hotword service layer - - TTS or voice-output ownership - broad assistant-platform scope - `SYSTRAN/faster-whisper` is now landed as a partially incorporated row rather than a generic future placeholder: @@ -311,7 +319,8 @@ Queue interpretation after that control pass: - orchestration-aware transcript metadata - bounded Python service-lane capability exposure - still deferred: - - microphone capture shell + - richer live service-lane batching, prompt-routing, or retrieval tuning beyond the landed + bounded profile - provider-profile and BYOK custody - downloadable model or payload shipping - TTS or voice-output ownership @@ -347,9 +356,9 @@ Queue interpretation after that control pass: - viseme / gesture runtime integration - broad assistant-platform scope - the next queue shape is now: - - retained `whisper.cpp` live microphone capture shell assessment + - retained `whisper.cpp` device-permission and capture-route readiness shell assessment - the next bounded move should stay narrow: - - a source-backed `Phase 6R-U` control pass before any widening into downloadable + - a source-backed `Phase 6R-V` control pass before any widening into downloadable model/payload shipping, broad voice-output ownership, or broad assistant-platform scope - keep the speech-input / voice sidecar legal sequencing guard visible: - keep code-license judgments separate from model, voice, and payload-license review