From 8671f3e6fa082f4aaead3fb0c9e173cff5a7039d Mon Sep 17 00:00:00 2001 From: axiomlogicnexus Date: Sun, 24 May 2026 20:55:42 +0200 Subject: [PATCH] Implement Phase 6R-V whisper.cpp permission readiness shell --- .../HyperTwistContractLibrary.cpp | 21 +- .../HyperTwistHttpSpeechClient.cpp | 9 + .../HyperTwistMockSpeechClient.cpp | 3 + .../HyperTwistTrainingSubsystem.cpp | 257 +++++++++++++++++- .../HyperTwistRecognitionTypes.h | 108 ++++++++ ...hase6RVPermissionReadinessContractTest.cpp | 248 +++++++++++++++++ docs/REPO_LICENSE_TRACKING.md | 20 +- ..._SHELL_IMPLEMENTATION_PACKET_2026-05-24.md | 162 +++++++++++ ...ESS_SHELL_PREPARATION_PACKET_2026-05-24.md | 166 +++++++++++ ...PERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md | 14 +- .../HyperTwist/FEATURE_REGISTRY.md | 4 +- .../HyperTwist/ROADMAP.md | 28 +- 12 files changed, 1009 insertions(+), 31 deletions(-) create mode 100644 UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RVPermissionReadinessContractTest.cpp create mode 100644 docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md create mode 100644 docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_PREPARATION_PACKET_2026-05-24.md diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp index 386dadf..b6455a9 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp @@ -850,12 +850,18 @@ namespace HyperTwistContractLibraryInternal Profile.StepWindowMs = 3000; Profile.CaptureWindowMs = 10000; Profile.KeepWindowMs = 200; - Profile.AudioContextTokens = 0; - Profile.VadThreshold = 0.6f; - Profile.HighPassFrequencyHz = 100.0f; - Profile.bKeepContextBetweenChunks = false; - Profile.bSupportsStreamingPreview = true; - Profile.bSupportsManualCommit = true; + Profile.AudioContextTokens = 0; + Profile.VadThreshold = 0.6f; + Profile.HighPassFrequencyHz = 100.0f; + Profile.bKeepContextBetweenChunks = false; + Profile.bSupportsStreamingPreview = true; + Profile.bSupportsManualCommit = true; + Profile.PermissionContractId = TEXT("device-permission-shell-v1"); + Profile.CaptureRouteContractId = TEXT("capture-route-readiness-shell-v1"); + Profile.PermissionRequestActionId = TEXT("request-microphone-permission"); + Profile.CaptureRouteRetryActionId = TEXT("retry-capture-route"); + Profile.bSupportsPermissionRequest = true; + Profile.bSupportsCaptureRouteRetry = true; auto AddPanel = [&Profile]( const TCHAR* PanelId, @@ -1724,6 +1730,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechService TEXT("grammarHints"), TEXT("sessioned-mock-client"), TEXT("microphoneCaptureShell"), + TEXT("devicePermissionShell"), + TEXT("captureRouteReadinessShell"), + TEXT("captureRouteRetry"), TEXT("streamingPreview"), TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp index 12c5212..3519601 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp @@ -318,6 +318,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal TEXT("vadSegments"), TEXT("grammarHints"), TEXT("microphoneCaptureShell"), + TEXT("devicePermissionShell"), + TEXT("captureRouteReadinessShell"), + TEXT("captureRouteRetry"), TEXT("streamingPreview"), TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), @@ -357,6 +360,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal TEXT("vadSegments"), TEXT("grammarHints"), TEXT("microphoneCaptureShell"), + TEXT("devicePermissionShell"), + TEXT("captureRouteReadinessShell"), + TEXT("captureRouteRetry"), TEXT("streamingPreview"), TEXT("manualTranscriptCommit"), TEXT("batchedTranscribe"), @@ -376,6 +382,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal Health.ProviderLabel = ProviderLabel; } Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell")); + Health.Capabilities.AddUnique(TEXT("devicePermissionShell")); + Health.Capabilities.AddUnique(TEXT("captureRouteReadinessShell")); + Health.Capabilities.AddUnique(TEXT("captureRouteRetry")); Health.Capabilities.AddUnique(TEXT("streamingPreview")); Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit")); Health.SupportedOrchestrationProfiles.AddUnique(TEXT("python-batch-transcribe-v1")); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp index ea36744..4619f9a 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp @@ -129,6 +129,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHeal Health.ServiceEndpoint = TEXT("in-process://mock"); Health.Capabilities.AddUnique(TEXT("offline-transcribe")); Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell")); + Health.Capabilities.AddUnique(TEXT("devicePermissionShell")); + Health.Capabilities.AddUnique(TEXT("captureRouteReadinessShell")); + Health.Capabilities.AddUnique(TEXT("captureRouteRetry")); Health.Capabilities.AddUnique(TEXT("streamingPreview")); Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit")); Health.Capabilities.AddUnique(TEXT("batchedTranscribe")); diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp index 9b2928f..5d98209 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp @@ -2910,6 +2910,12 @@ namespace HyperTwistTrainingSubsystemInternal return BrowserShellState; } + FHyperTwistSpeechMicrophoneShellState BuildCoachCommandMicrophoneShellState( + const FHyperTwistSpeechSessionConfig& SessionConfig, + const FHyperTwistTrainingCompanionSpeechSessionState* PriorState, + const bool bSessionOpen + ); + FHyperTwistSpeechMicrophoneShellProfile BuildDefaultCoachCommandMicrophoneShellProfile(const FString& ProfileId) { FHyperTwistSpeechMicrophoneShellProfile Profile; @@ -2925,6 +2931,12 @@ namespace HyperTwistTrainingSubsystemInternal Profile.bKeepContextBetweenChunks = false; Profile.bSupportsStreamingPreview = true; Profile.bSupportsManualCommit = true; + Profile.PermissionContractId = TEXT("device-permission-shell-v1"); + Profile.CaptureRouteContractId = TEXT("capture-route-readiness-shell-v1"); + Profile.PermissionRequestActionId = TEXT("request-microphone-permission"); + Profile.CaptureRouteRetryActionId = TEXT("retry-capture-route"); + Profile.bSupportsPermissionRequest = true; + Profile.bSupportsCaptureRouteRetry = true; auto AddPanel = [&Profile]( const TCHAR* PanelId, @@ -2966,6 +2978,225 @@ namespace HyperTwistTrainingSubsystemInternal return Profile; } + bool IsSpeechPermissionError(const FString& ErrorText) + { + const FString NormalizedError = ErrorText.ToLower(); + return NormalizedError.Contains(TEXT("permission")) + || NormalizedError.Contains(TEXT("not-allowed")) + || NormalizedError.Contains(TEXT("access-denied")); + } + + bool IsSpeechRouteError(const FString& ErrorText) + { + const FString NormalizedError = ErrorText.ToLower(); + return NormalizedError.Contains(TEXT("speech-client")) + || NormalizedError.Contains(TEXT("service-base-url")) + || NormalizedError.Contains(TEXT("request-")) + || NormalizedError.Contains(TEXT("response-missing")) + || NormalizedError.Contains(TEXT("invalid-provider-response")) + || NormalizedError.Contains(TEXT("http-")) + || NormalizedError.Contains(TEXT("session-not-open")); + } + + void AddSpeechMicrophoneShellIssue( + FHyperTwistSpeechMicrophoneShellState& ShellState, + const FHyperTwistSpeechMicrophoneShellIssue& Issue + ) + { + if (!Issue.IsStructurallyValid()) + { + return; + } + + ShellState.Issues.Add(Issue); + if (!ShellState.ActiveIssue.IsStructurallyValid()) + { + ShellState.ActiveIssue = Issue; + } + } + + void ApplySpeechMicrophoneShellPosture( + const FHyperTwistSpeechMicrophoneShellProfile& Profile, + const FHyperTwistSpeechServiceHealth& ServiceHealth, + const FString& LastError, + const bool bSessionOpen, + FHyperTwistSpeechMicrophoneShellState& ShellState + ) + { + ShellState.PermissionStateId = TEXT("permission-granted"); + ShellState.CaptureRouteStateId = bSessionOpen ? TEXT("route-ready") : TEXT("route-closed"); + ShellState.StatusLine = bSessionOpen + ? TEXT("Microphone route ready for coach commands.") + : TEXT("Microphone session closed."); + ShellState.DetailLine = bSessionOpen + ? TEXT("Live capture is armed for the next bounded coach command.") + : TEXT("Reopen the companion speech session when microphone capture is needed again."); + ShellState.bMicrophonePermissionGranted = true; + ShellState.bCaptureRouteReady = bSessionOpen && ServiceHealth.bReady; + ShellState.bPermissionPromptVisible = false; + ShellState.bPermissionRequestVisible = false; + ShellState.bCaptureRouteRetryVisible = false; + ShellState.Issues.Reset(); + ShellState.ActiveIssue = FHyperTwistSpeechMicrophoneShellIssue(); + ShellState.ActiveIssueCount = 0; + + if (!bSessionOpen) + { + ShellState.bCaptureActive = false; + ShellState.bAwaitingSpeech = false; + ShellState.bTranscriptionInFlight = false; + } + + const FString RouteError = !LastError.IsEmpty() ? LastError : ServiceHealth.LastError; + ShellState.LastRouteError = RouteError; + if (LastError == TEXT("no-active-training-run")) + { + ShellState.CaptureRouteStateId = TEXT("route-idle"); + ShellState.StatusLine = TEXT("Start a training run before microphone capture opens."); + ShellState.DetailLine = + TEXT("The shell is available, but no active training run currently owns the speech route."); + ShellState.bCaptureRouteReady = false; + } + else if (IsSpeechPermissionError(RouteError)) + { + FHyperTwistSpeechMicrophoneShellIssue Issue; + Issue.IssueId = TEXT("microphone-permission-required"); + Issue.IssueKind = TEXT("permission-required"); + Issue.StatusLine = TEXT("Microphone permission is required before capture can start."); + Issue.DetailLine = + TEXT("Grant microphone access and reopen the shell. This packet only exposes shell posture; it does not own the OS permission workflow."); + Issue.RecommendedActionId = Profile.PermissionRequestActionId; + AddSpeechMicrophoneShellIssue(ShellState, Issue); + + ShellState.PermissionStateId = TEXT("permission-required"); + ShellState.CaptureRouteStateId = TEXT("route-waiting-for-permission"); + ShellState.StatusLine = Issue.StatusLine; + ShellState.DetailLine = Issue.DetailLine; + ShellState.bMicrophonePermissionGranted = false; + ShellState.bCaptureRouteReady = false; + ShellState.bCaptureActive = false; + ShellState.bAwaitingSpeech = false; + ShellState.bTranscriptionInFlight = false; + ShellState.bPermissionPromptVisible = Profile.bSupportsPermissionRequest; + ShellState.bPermissionRequestVisible = Profile.bSupportsPermissionRequest; + } + else if ((IsSpeechRouteError(RouteError) && LastError != TEXT("no-active-training-run")) || (bSessionOpen && !ServiceHealth.bReady)) + { + FHyperTwistSpeechMicrophoneShellIssue Issue; + Issue.IssueId = TEXT("capture-route-unavailable"); + Issue.IssueKind = TEXT("capture-route-unavailable"); + Issue.StatusLine = TEXT("Capture route unavailable."); + Issue.DetailLine = RouteError.IsEmpty() + ? TEXT("The speech route is not ready yet. Retry once the sidecar or device route reports healthy again.") + : FString::Printf( + TEXT("The speech route reported '%s'. Retry once the sidecar or device route reports healthy again."), + *RouteError + ); + Issue.RecommendedActionId = Profile.CaptureRouteRetryActionId; + AddSpeechMicrophoneShellIssue(ShellState, Issue); + + ShellState.CaptureRouteStateId = TEXT("route-retry-required"); + ShellState.StatusLine = Issue.StatusLine; + ShellState.DetailLine = Issue.DetailLine; + ShellState.bCaptureRouteReady = false; + ShellState.bCaptureActive = false; + ShellState.bAwaitingSpeech = false; + ShellState.bTranscriptionInFlight = false; + ShellState.bCaptureRouteRetryVisible = Profile.bSupportsCaptureRouteRetry; + } + + if (ShellState.bPermissionRequestVisible) + { + ShellState.AvailableActionIds.AddUnique(Profile.PermissionRequestActionId); + } + if (ShellState.bCaptureRouteRetryVisible) + { + ShellState.AvailableActionIds.AddUnique(Profile.CaptureRouteRetryActionId); + } + + ShellState.ActiveIssueCount = ShellState.Issues.Num(); + } + + void SynchronizeSpeechMicrophoneShellState(FHyperTwistTrainingCompanionSpeechSessionState& SessionState) + { + FHyperTwistSpeechSessionConfig EffectiveConfig = SessionState.SessionConfig; + if (EffectiveConfig.ListeningContractId.IsEmpty()) + { + EffectiveConfig.ListeningContractId = TEXT("listening-threshold-lifecycle"); + } + if (EffectiveConfig.InputRouteId.IsEmpty()) + { + EffectiveConfig.InputRouteId = TEXT("queue/listening"); + } + if (EffectiveConfig.LanguageMode.IsEmpty()) + { + EffectiveConfig.LanguageMode = TEXT("auto"); + } + if (EffectiveConfig.MicrophoneShellProfileId.IsEmpty()) + { + EffectiveConfig.MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1"); + } + if (!EffectiveConfig.bEnableVad) + { + EffectiveConfig.bEnableVad = true; + } + const FHyperTwistSpeechMicrophoneShellProfile Profile = + EffectiveConfig.MicrophoneShellProfileDefinition.IsStructurallyValid() + ? EffectiveConfig.MicrophoneShellProfileDefinition + : BuildDefaultCoachCommandMicrophoneShellProfile( + !EffectiveConfig.MicrophoneShellProfileId.IsEmpty() + ? EffectiveConfig.MicrophoneShellProfileId + : TEXT("coach-command-microphone-shell-v1") + ); + + if (!SessionState.MicrophoneShellState.IsStructurallyValid()) + { + SessionState.MicrophoneShellState = + BuildCoachCommandMicrophoneShellState( + EffectiveConfig, + &SessionState, + SessionState.bSessionOpen + ); + } + + FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState; + ShellState.MicrophoneShellProfileId = Profile.MicrophoneShellProfileId; + ShellState.CaptureMode = Profile.CaptureMode; + ShellState.ListeningContractId = !EffectiveConfig.ListeningContractId.IsEmpty() + ? EffectiveConfig.ListeningContractId + : TEXT("listening-threshold-lifecycle"); + ShellState.InputRouteId = !EffectiveConfig.InputRouteId.IsEmpty() + ? EffectiveConfig.InputRouteId + : TEXT("queue/listening"); + if (ShellState.LastLanguageCode.IsEmpty()) + { + ShellState.LastLanguageCode = + EffectiveConfig.LanguageMode.IsEmpty() + || EffectiveConfig.LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase) + ? TEXT("en") + : EffectiveConfig.LanguageMode; + } + ShellState.StepWindowMs = Profile.StepWindowMs; + ShellState.CaptureWindowMs = Profile.CaptureWindowMs; + ShellState.KeepWindowMs = Profile.KeepWindowMs; + ShellState.VadThreshold = Profile.VadThreshold; + ShellState.bSessionOpen = SessionState.bSessionOpen; + ShellState.bVadArmed = EffectiveConfig.bEnableVad; + ShellState.AvailableActionIds.Reset(); + for (const FHyperTwistSpeechShellActionBinding& ActionBinding : Profile.ActionBindings) + { + ShellState.AvailableActionIds.AddUnique(ActionBinding.ActionId); + } + + ApplySpeechMicrophoneShellPosture( + Profile, + SessionState.ServiceHealth, + SessionState.LastError, + SessionState.bSessionOpen, + ShellState + ); + } + FHyperTwistSpeechMicrophoneShellState BuildCoachCommandMicrophoneShellState( const FHyperTwistSpeechSessionConfig& SessionConfig, const FHyperTwistTrainingCompanionSpeechSessionState* PriorState, @@ -2991,9 +3222,18 @@ namespace HyperTwistTrainingSubsystemInternal ? SessionConfig.InputRouteId : TEXT("queue/listening"); ShellState.LastLanguageCode = - SessionConfig.LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase) + SessionConfig.LanguageMode.IsEmpty() + || SessionConfig.LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase) ? TEXT("en") : SessionConfig.LanguageMode; + ShellState.PermissionStateId = TEXT("permission-granted"); + ShellState.CaptureRouteStateId = bSessionOpen ? TEXT("route-ready") : TEXT("route-closed"); + ShellState.StatusLine = bSessionOpen + ? TEXT("Microphone route ready for coach commands.") + : TEXT("Microphone session closed."); + ShellState.DetailLine = bSessionOpen + ? TEXT("Live capture is armed for the next bounded coach command.") + : TEXT("Reopen the companion speech session when microphone capture is needed again."); ShellState.StepWindowMs = Profile.StepWindowMs; ShellState.CaptureWindowMs = Profile.CaptureWindowMs; ShellState.KeepWindowMs = Profile.KeepWindowMs; @@ -8399,6 +8639,7 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out } const FHyperTwistSpeechSessionConfig SessionConfig = BuildActiveCompanionSpeechSessionConfig(); + ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig; if (SessionConfig.SessionId.IsEmpty()) { OutError = TEXT("speech-session-id-unavailable"); @@ -8420,7 +8661,7 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out true ); ActiveCompanionSpeechSessionState.LastError.Reset(); - ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset(); + RefreshCompanionSpeechServiceHealth(); return true; } @@ -8491,6 +8732,9 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp Result.Warnings.Add(TEXT("speech-client-unavailable")); ActiveCompanionSpeechSessionState.LastError = Result.Warnings[0]; ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError; + HyperTwistTrainingSubsystemInternal::SynchronizeSpeechMicrophoneShellState( + ActiveCompanionSpeechSessionState + ); return Result; } @@ -8659,6 +8903,9 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = false; ActiveCompanionSpeechSessionState.MicrophoneShellState.bCommitReady = Result.bIsFinal && Result.IsStructurallyValid(); + HyperTwistTrainingSubsystemInternal::SynchronizeSpeechMicrophoneShellState( + ActiveCompanionSpeechSessionState + ); return Result; } @@ -9360,10 +9607,16 @@ void UHyperTwistTrainingSubsystem::RefreshCompanionSpeechServiceHealth() : TEXT("in-process://mock"); ActiveCompanionSpeechSessionState.ServiceHealth.bReady = false; ActiveCompanionSpeechSessionState.ServiceHealth.LastError = TEXT("speech-client-unavailable"); + HyperTwistTrainingSubsystemInternal::SynchronizeSpeechMicrophoneShellState( + ActiveCompanionSpeechSessionState + ); return; } ActiveCompanionSpeechSessionState.ServiceHealth = SpeechClient->GetSpeechServiceHealth(); + HyperTwistTrainingSubsystemInternal::SynchronizeSpeechMicrophoneShellState( + ActiveCompanionSpeechSessionState + ); } void UHyperTwistTrainingSubsystem::AppendRecognitionReplayEvent( diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h index 0d61bee..a5e8f67 100644 --- a/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h @@ -1522,6 +1522,24 @@ struct FHyperTwistSpeechMicrophoneShellProfile UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") bool bSupportsManualCommit = true; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString PermissionContractId = TEXT("device-permission-shell-v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CaptureRouteContractId = TEXT("capture-route-readiness-shell-v1"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString PermissionRequestActionId = TEXT("request-microphone-permission"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CaptureRouteRetryActionId = TEXT("retry-capture-route"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSupportsPermissionRequest = true; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bSupportsCaptureRouteRetry = true; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") TArray Panels; @@ -1540,6 +1558,10 @@ struct FHyperTwistSpeechMicrophoneShellProfile || AudioContextTokens < 0 || VadThreshold <= 0.0f || HighPassFrequencyHz < 0.0f + || PermissionContractId.IsEmpty() + || CaptureRouteContractId.IsEmpty() + || PermissionRequestActionId.IsEmpty() + || CaptureRouteRetryActionId.IsEmpty() || Panels.Num() <= 0 || ActionBindings.Num() <= 0) { @@ -1566,6 +1588,39 @@ struct FHyperTwistSpeechMicrophoneShellProfile } }; +USTRUCT(BlueprintType) +struct FHyperTwistSpeechMicrophoneShellIssue +{ + GENERATED_BODY() + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString IssueId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString IssueKind; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString StatusLine; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString DetailLine; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString RecommendedActionId; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bBlocksCaptureStart = true; + + bool IsStructurallyValid() const + { + return !IssueId.IsEmpty() + && !IssueKind.IsEmpty() + && !StatusLine.IsEmpty() + && !DetailLine.IsEmpty() + && !RecommendedActionId.IsEmpty(); + } +}; + USTRUCT(BlueprintType) struct FHyperTwistSpeechMicrophoneShellState { @@ -1583,6 +1638,12 @@ struct FHyperTwistSpeechMicrophoneShellState UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FString InputRouteId = TEXT("queue/listening"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString PermissionStateId = TEXT("permission-granted"); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString CaptureRouteStateId = TEXT("route-ready"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FString ActiveUtteranceId; @@ -1595,6 +1656,15 @@ struct FHyperTwistSpeechMicrophoneShellState UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") FString LastLanguageCode = TEXT("en"); + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString StatusLine = TEXT("Microphone route ready for coach commands."); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString DetailLine = TEXT("Speak a short coach command when the shell is armed."); + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FString LastRouteError; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") int32 StepWindowMs = 3000; @@ -1661,6 +1731,21 @@ struct FHyperTwistSpeechMicrophoneShellState UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") bool bPermissionPromptVisible = false; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bPermissionRequestVisible = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + bool bCaptureRouteRetryVisible = false; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + int32 ActiveIssueCount = 0; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + FHyperTwistSpeechMicrophoneShellIssue ActiveIssue; + + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") + TArray Issues; + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist") TArray AvailableActionIds; @@ -1670,7 +1755,11 @@ struct FHyperTwistSpeechMicrophoneShellState || CaptureMode.IsEmpty() || ListeningContractId.IsEmpty() || InputRouteId.IsEmpty() + || PermissionStateId.IsEmpty() + || CaptureRouteStateId.IsEmpty() || LastLanguageCode.IsEmpty() + || StatusLine.IsEmpty() + || DetailLine.IsEmpty() || StepWindowMs <= 0 || CaptureWindowMs < StepWindowMs || KeepWindowMs < 0 @@ -1681,11 +1770,30 @@ struct FHyperTwistSpeechMicrophoneShellState || DetectedSpeechEndMs < DetectedSpeechStartMs || LastSilenceGapMs < 0 || VadThreshold <= 0.0f + || ActiveIssueCount < 0 || AvailableActionIds.Num() <= 0) { return false; } + if (ActiveIssueCount != Issues.Num()) + { + return false; + } + + if (ActiveIssueCount > 0 && !ActiveIssue.IsStructurallyValid()) + { + return false; + } + + for (const FHyperTwistSpeechMicrophoneShellIssue& Issue : Issues) + { + if (!Issue.IsStructurallyValid()) + { + return false; + } + } + for (const FString& ActionId : AvailableActionIds) { if (ActionId.IsEmpty()) diff --git a/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RVPermissionReadinessContractTest.cpp b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RVPermissionReadinessContractTest.cpp new file mode 100644 index 0000000..d396b60 --- /dev/null +++ b/UnrealHyperTwist/Source/UnrealHyperTwist/Tests/HyperTwistWhisperCppPhase6RVPermissionReadinessContractTest.cpp @@ -0,0 +1,248 @@ +// Copyright HyperTwist, Inc. All Rights Reserved. + +#include "Misc/AutomationTest.h" + +#include "Engine/GameInstance.h" +#include "HyperTwistRecognition/HyperTwistSpeechClient.h" +#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h" +#include "UObject/UnrealType.h" + +#if WITH_AUTOMATION_TESTS + +namespace HyperTwistWhisperCppPhase6RVTestInternal +{ + FHyperTwistTrainingDeck MakeSpeechDeck() + { + FHyperTwistTrainingDeck Deck; + Deck.DeckId = TEXT("phase6r-v/whispercpp-permission-readiness"); + Deck.Title = TEXT("Phase 6R-V Whisper Permission Readiness"); + Deck.DeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + + FHyperTwistTrainingCase TrainingCase; + TrainingCase.CaseId = TEXT("phase6r-v-case"); + TrainingCase.PuzzleId = TEXT("cube/3x3x3"); + TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence; + TrainingCase.PromptLabel = TEXT("Phase 6R-V Coach Speech"); + TrainingCase.AllowedDeliveryModes = { + EHyperTwistTrainingDeliveryMode::CoachReviewed + }; + Deck.Cases = {TrainingCase}; + return Deck; + } + + void ForceSpeechClientKind(UHyperTwistTrainingSubsystem* TrainingSubsystem, const FString& ClientKind) + { + if (TrainingSubsystem == nullptr) + { + return; + } + + if (FStrProperty* SpeechClientKindProperty = FindFProperty( + UHyperTwistTrainingSubsystem::StaticClass(), + TEXT("CompanionSpeechClientKind") + )) + { + SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, ClientKind); + } + } + + UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId, const FString& ClientKind) + { + UGameInstance* GameInstance = NewObject(GetTransientPackage()); + if (GameInstance == nullptr) + { + return nullptr; + } + + UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject(GameInstance); + if (TrainingSubsystem == nullptr) + { + return nullptr; + } + + ForceSpeechClientKind(TrainingSubsystem, ClientKind); + const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck( + MakeSpeechDeck(), + TEXT("phase6r-v-user"), + SessionId, + EHyperTwistTrainingDeliveryMode::CoachReviewed + ); + return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr; + } + + UHyperTwistHttpSpeechClient* ResolveHttpSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem) + { + if (TrainingSubsystem == nullptr) + { + return nullptr; + } + + FString CloseError; + TrainingSubsystem->CloseActiveCompanionSpeechSession(CloseError); + + if (FObjectPropertyBase* ClientProperty = FindFProperty( + UHyperTwistTrainingSubsystem::StaticClass(), + TEXT("ActiveCompanionSpeechClientObject") + )) + { + return Cast( + ClientProperty->GetObjectPropertyValue_InContainer(TrainingSubsystem) + ); + } + + return nullptr; + } +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RVPermissionReadinessSessionConfigTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.V.PermissionReadinessSessionConfig", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RVPermissionReadinessSessionConfigTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RVTestInternal::MakeSpeechSubsystem( + TEXT("phase6r-v-config-session"), + TEXT("mock") + ); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-V."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue(TEXT("The permission/readiness shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellProfile& Profile = + SessionState.SessionConfig.MicrophoneShellProfileDefinition; + + TestTrue(TEXT("The permission/readiness route must expose a structurally valid shell profile."), Profile.IsStructurallyValid()); + TestEqual(TEXT("The shell profile must preserve the permission contract id."), Profile.PermissionContractId, TEXT("device-permission-shell-v1")); + TestEqual(TEXT("The shell profile must preserve the capture-route contract id."), Profile.CaptureRouteContractId, TEXT("capture-route-readiness-shell-v1")); + TestEqual(TEXT("The shell profile must preserve the permission request action id."), Profile.PermissionRequestActionId, TEXT("request-microphone-permission")); + TestEqual(TEXT("The shell profile must preserve the capture-route retry action id."), Profile.CaptureRouteRetryActionId, TEXT("retry-capture-route")); + TestTrue(TEXT("The shell profile must advertise permission-request support."), Profile.bSupportsPermissionRequest); + TestTrue(TEXT("The shell profile must advertise capture-route retry support."), Profile.bSupportsCaptureRouteRetry); + TestTrue( + TEXT("The speech service health must expose device-permission shell capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("devicePermissionShell")) + ); + TestTrue( + TEXT("The speech service health must expose capture-route readiness shell capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("captureRouteReadinessShell")) + ); + TestTrue( + TEXT("The speech service health must expose capture-route retry capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("captureRouteRetry")) + ); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RVPermissionReadinessOpenStateTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.V.PermissionReadinessOpenState", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RVPermissionReadinessOpenStateTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RVTestInternal::MakeSpeechSubsystem( + TEXT("phase6r-v-open-session"), + TEXT("mock") + ); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-V."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + FString OpenError; + TestTrue(TEXT("The permission/readiness shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty()); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState; + + TestTrue(TEXT("The permission/readiness open state must be structurally valid."), ShellState.IsStructurallyValid()); + TestEqual(TEXT("The open state must preserve granted permission posture."), ShellState.PermissionStateId, TEXT("permission-granted")); + TestEqual(TEXT("The open state must preserve route-ready posture."), ShellState.CaptureRouteStateId, TEXT("route-ready")); + TestEqual(TEXT("The open state must expose the ready status line."), ShellState.StatusLine, TEXT("Microphone route ready for coach commands.")); + TestTrue(TEXT("The open state must keep microphone permission granted."), ShellState.bMicrophonePermissionGranted); + TestTrue(TEXT("The open state must keep the capture route ready."), ShellState.bCaptureRouteReady); + TestFalse(TEXT("The open state must keep the permission prompt hidden."), ShellState.bPermissionPromptVisible); + TestFalse(TEXT("The open state must not surface a permission-request action."), ShellState.bPermissionRequestVisible); + TestFalse(TEXT("The open state must not surface a capture-route retry action."), ShellState.bCaptureRouteRetryVisible); + TestEqual(TEXT("The open state must not report active shell issues."), ShellState.ActiveIssueCount, 0); + TestTrue(TEXT("The open state must keep the bounded shell action count unchanged."), ShellState.AvailableActionIds.Num() == 4); + return true; +} + +IMPLEMENT_SIMPLE_AUTOMATION_TEST( + FHyperTwistWhisperCppPhase6RVCaptureRouteFallbackStateTest, + "HyperTwist.Permissive.WhisperCpp.Phase6R.V.CaptureRouteFallbackState", + EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter +) + +bool FHyperTwistWhisperCppPhase6RVCaptureRouteFallbackStateTest::RunTest(const FString& Parameters) +{ + UHyperTwistTrainingSubsystem* TrainingSubsystem = + HyperTwistWhisperCppPhase6RVTestInternal::MakeSpeechSubsystem( + TEXT("phase6r-v-route-session"), + TEXT("http-sidecar") + ); + TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-V."), TrainingSubsystem); + if (TrainingSubsystem == nullptr) + { + return false; + } + + UHyperTwistHttpSpeechClient* HttpSpeechClient = + HyperTwistWhisperCppPhase6RVTestInternal::ResolveHttpSpeechClient(TrainingSubsystem); + TestNotNull(TEXT("The provider-backed speech client must be available for the readiness fallback test."), HttpSpeechClient); + if (HttpSpeechClient == nullptr) + { + return false; + } + + HttpSpeechClient->ServiceBaseUrl.Reset(); + + FString OpenError; + TestFalse(TEXT("The readiness fallback route must fail cleanly when no provider endpoint is configured."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError)); + TestEqual(TEXT("The fallback route must report the missing service base URL."), OpenError, TEXT("service-base-url-missing")); + + const FHyperTwistTrainingCompanionSpeechSessionState SessionState = + TrainingSubsystem->GetActiveCompanionSpeechSessionState(); + const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState; + + TestTrue(TEXT("The readiness fallback shell state must remain structurally valid."), ShellState.IsStructurallyValid()); + TestEqual(TEXT("The fallback shell must keep permission granted when the failure is route-only."), ShellState.PermissionStateId, TEXT("permission-granted")); + TestEqual(TEXT("The fallback shell must surface route retry posture."), ShellState.CaptureRouteStateId, TEXT("route-retry-required")); + TestFalse(TEXT("The fallback shell must report the capture route as not ready."), ShellState.bCaptureRouteReady); + TestFalse(TEXT("The fallback shell must keep the permission prompt hidden for route failures."), ShellState.bPermissionPromptVisible); + TestTrue(TEXT("The fallback shell must surface capture-route retry."), ShellState.bCaptureRouteRetryVisible); + TestEqual(TEXT("The fallback shell must carry the route error."), ShellState.LastRouteError, TEXT("service-base-url-missing")); + TestEqual(TEXT("The fallback shell must report one active shell issue."), ShellState.ActiveIssueCount, 1); + TestEqual(TEXT("The fallback shell must classify the issue as capture-route unavailable."), ShellState.ActiveIssue.IssueKind, TEXT("capture-route-unavailable")); + TestEqual(TEXT("The fallback shell must recommend route retry."), ShellState.ActiveIssue.RecommendedActionId, TEXT("retry-capture-route")); + TestTrue( + TEXT("The provider-backed health must still advertise device-permission shell capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("devicePermissionShell")) + ); + TestTrue( + TEXT("The provider-backed health must still advertise capture-route readiness shell capability."), + SessionState.ServiceHealth.Capabilities.Contains(TEXT("captureRouteReadinessShell")) + ); + return true; +} + +#endif diff --git a/docs/REPO_LICENSE_TRACKING.md b/docs/REPO_LICENSE_TRACKING.md index e58a0e4..95101e8 100644 --- a/docs/REPO_LICENSE_TRACKING.md +++ b/docs/REPO_LICENSE_TRACKING.md @@ -171,9 +171,13 @@ Status update on `2026-05-21`: control pass is now consumed - the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell packet is now landed in current code -- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` - device-permission and capture-route readiness shell preparation/control pass, not a new - restrictive packet by default +- the generic source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell control pass is now consumed +- the bounded permissive `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell packet is now landed in current code +- the current next bounded move is a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` + downloadable model and payload custody preparation/control pass, not a new restrictive packet + by default - use the repo-row README census, the portfolio standing refresh backfill, and the `2R-A` ownership contract for the current queue after that correction @@ -277,8 +281,8 @@ Current practical interpretation: - the landed `PostHog/posthog` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_D_POSTHOG_CONTROL_PLANE_IMPLEMENTATION_2026-05-13.md` - the landed `screenpipe/screenpipe` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_E_SCREENPIPE_CAPTURE_HISTORY_IMPLEMENTATION_2026-05-13.md` - the landed `remotion-dev/remotion` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_F_REMOTION_MEDIA_EXPORT_IMPLEMENTATION_2026-05-13.md` -- the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` - device-permission and capture-route readiness shell preparation/control pass +- the next bounded move is a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` + downloadable model and payload custody preparation/control pass Companion docs: @@ -1528,10 +1532,12 @@ Approved working posture: - the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle - the next bounded permissive implementation slice is now landed as a live microphone shell profile and shell-state boundary above the existing transcript-session seam +- the next bounded permissive implementation slice is now landed as device-permission and + capture-route readiness shell posture above the existing microphone shell seam - the top-level provider/session contract remains first-party HyperTwist-owned and provider-neutral; this row does not own that lane -- keep real device-permission workflow, model files, downloadable payloads, and future - voice-asset review separate from the code-license judgment +- keep real device-permission workflow, native capture-route ownership, model files, + downloadable payloads, and future voice-asset review separate from the code-license judgment - treat `SYSTRAN/faster-whisper` as the landed complementary Python-orchestration donor rather than widening `whisper.cpp` into the runtime core diff --git a/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md b/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md new file mode 100644 index 0000000..0805b7f --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md @@ -0,0 +1,162 @@ +# HyperTwist Phase 6R-V whisper.cpp device-permission and capture-route readiness shell implementation packet + +Created on `2026-05-24` + +## Status + +- first-party HyperTwist packet +- bounded permissive `Phase 6R-V` implementation slice + +## Purpose + +This packet lands the next narrower bounded slice from the retained +`ggml-org/whisper.cpp` speech-input row. + +The landed slice is: + +- first-party device-permission and capture-route readiness shell posture above + the existing live microphone shell seam + +It is not: + +- a full `whisper.cpp` row transplant +- a real device-permission workflow packet +- a native audio-device route ownership packet +- a downloadable model or payload-shipping packet +- a top-level provider/session contract packet +- a voice-output packet +- a broad assistant-platform packet + +## Current authority basis + +This implementation packet stands on: + +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_PREPARATION_PACKET_2026-05-24.md` + +The retained owner remains: + +- `ggml-org/whisper.cpp` + +The top-level provider/session contract owner does not change here: + +- first-party HyperTwist remains the provider-neutral owner for the speech + provider/session lane +- `SYSTRAN/faster-whisper` remains the complementary landed Python + orchestration donor +- this packet keeps `ggml-org/whisper.cpp` bounded to the narrower + permission/readiness shell donor slice only + +The granted family remains bounded to: + +- explicit permission/readiness shell contract shaping +- route fallback issue-object surfacing +- lifecycle posture sync on top of the landed microphone shell seam +- bounded speech-health capability exposure for the narrower shell + +This packet lands only the next narrower family in that granted set. + +## Landed scope + +The current code now owns a retained permission/readiness shell contract +through: + +- retained recognition contract types for: + - `FHyperTwistSpeechMicrophoneShellIssue` + - expanded `FHyperTwistSpeechMicrophoneShellProfile` + - expanded `FHyperTwistSpeechMicrophoneShellState` +- sample session-config and service-health outputs in: + - `UHyperTwistContractLibrary` +- direct-donor speech client seam capability exposure in: + - `UHyperTwistMockSpeechClient` + - `UHyperTwistHttpSpeechClient` +- active companion shell-state posture synchronization in: + - `UHyperTwistTrainingSubsystem` +- focused automation coverage in: + - `HyperTwistWhisperCppPhase6RVPermissionReadinessContractTest.cpp` + +## Why this is still intentionally bounded + +This packet lands the next donor-backed shell seam, but it does not widen into +the neighboring retained families. + +Still deferred: + +- real device-permission workflow +- native audio-device route ownership beyond bounded shell posture +- downloadable model or payload shipping +- provider-profile and BYOK custody +- voice-output ownership +- broad assistant-platform scope + +## Validation + +Build validation: + +- `C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE -NoUba` + +Focused automation validation: + +- `C:\Program Files\Epic Games\UE_5.7\Engine\Binaries\Win64\UnrealEditor-Cmd.exe C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject -unattended -nop4 -nosplash -NullRHI -log -stdout -FullStdOutLogOutput -AbsLog=C:\HyperTwist\UnrealHyperTwist\Saved\Logs\Phase6R-V-WhisperCpp-Verify.log -ReportExportPath=C:\HyperTwist\UnrealHyperTwist\Saved\AutomationReports\Phase6R-V-WhisperCpp-Verify -ExecCmds="Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.V; Quit" -TestExit="Automation Test Queue Empty"` + +Regression automation validation: + +- `HyperTwist.Permissive.WhisperCpp.Phase6R.U` +- `HyperTwist.Permissive.WhisperCpp.Phase6R.E` +- `HyperTwist.Permissive.FasterWhisper.Phase6R.F` +- `HyperTwist.Permissive.Piper.Phase6R.G` +- `HyperTwist.Permissive.Coqui.Phase6R.H` +- `HyperTwist.CleanRoom.CubeDesk` + +Expected covered tests: + +- `CaptureRouteFallbackState` +- `PermissionReadinessOpenState` +- `PermissionReadinessSessionConfig` +- existing `Phase 6R-U`, `Phase 6R-E`, `Phase 6R-F`, `Phase 6R-G`, + `Phase 6R-H`, and `CubeDesk` regression suites + +## Queue effect + +This packet consumes the current `Phase 6R-V` implementation slice. + +`ggml-org/whisper.cpp` remains only partially incorporated: + +- landed now: + - speech transcript session boundary + - VAD-aware session config and utterance envelope + - transcript segment and transcript result + - sidecar health and speech client boundary + - live microphone shell profile and shell-state boundary + - device-permission and capture-route readiness shell posture + - bounded shell capability exposure for: + - microphone capture shell + - device-permission shell + - capture-route readiness shell + - capture-route retry + - streaming preview + - manual transcript commit +- still deferred: + - real device-permission workflow + - native audio-device route ownership beyond bounded shell posture + - downloadable model or payload shipping + - provider-profile and BYOK custody + - broad assistant-platform scope + +The next clean move is: + +- a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` downloadable model and + payload custody preparation/control pass + +Keep the future sequencing guards visible: + +- `ggml-org/whisper.cpp` + - keep code-license judgments separate from downloadable model or payload + license review in any follow-on microphone packet +- first-party HyperTwist + - keep the top-level provider/session contract first-party rather than + letting `whisper.cpp` or `faster-whisper` absorb that lane through a + narrower donor win +- `SYSTRAN/faster-whisper` + - keep the row complementary to the landed shell and bounded to Python + orchestration/service-lane ownership rather than shell ownership diff --git a/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_PREPARATION_PACKET_2026-05-24.md b/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_PREPARATION_PACKET_2026-05-24.md new file mode 100644 index 0000000..c3d5278 --- /dev/null +++ b/docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_PREPARATION_PACKET_2026-05-24.md @@ -0,0 +1,166 @@ +# HyperTwist Phase 6R-V whisper.cpp device-permission and capture-route readiness shell preparation packet + +Created on `2026-05-24` + +## Status + +- historical same-day preparation authority +- bounded post-`Phase 6R-U` control slice +- the first bounded implementation slice now lands separately under: + - `docs/arch/HYPERTWIST_PHASE6R_V_WHISPER_CPP_DEVICE_PERMISSION_AND_CAPTURE_ROUTE_READINESS_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md` + +## Purpose + +This packet freezes the next widening order after the landed `Phase 6R-U` +`whisper.cpp` live microphone capture shell slice. + +The open task was: + +- define the next bounded source-backed widening packet for + `ggml-org/whisper.cpp` as the retained `Phase 6R-V` + device-permission and capture-route readiness shell seam + +It is not: + +- a full `whisper.cpp` row transplant +- a real device-permission workflow packet +- a native audio-device route ownership packet +- a downloadable model or payload-shipping packet +- a `SYSTRAN/faster-whisper` provider/service rewrite packet +- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet +- a broad assistant-platform packet + +## Current authority basis + +This preparation packet stands on already-closed authority: + +- `docs/REPO_LICENSE_TRACKING.md` +- `docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md` +- `docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md` + +The key accepted routing facts are: + +- `ggml-org/whisper.cpp` remains the retained offline STT donor after the + already landed `Phase 6R-E` transcript-session slice and the landed + `Phase 6R-U` microphone shell slice +- `SYSTRAN/faster-whisper` remains the complementary landed Python + orchestration donor above the existing first-party speech session boundary +- first-party HyperTwist already owns the top-level provider/session contract, + companion listening lifecycle, and speech-session substrate +- the retained donor value strongest for the next narrow widening is: + - explicit microphone-permission shell wording + - capture-route readiness / retry posture + - capture-device / route-selection failure surfacing + - bounded request / retry capability exposure +- ownership denied here is: + - do not widen into real device-permission workflow ownership + - do not widen into native audio-device route ownership beyond bounded shell + posture + - do not widen into downloadable model or payload shipping + - do not let `whisper.cpp` inherit the top-level provider/session lane + - do not widen into voice-output or broad assistant-platform ownership + +## Required result + +The source-backed control pass for this packet is now complete. + +The first actual `6R-V` implementation packet should: + +- define one bounded permission/readiness shell posture slice above the landed + microphone shell seam +- land first-party profile/state fields for explicit permission and route + readiness semantics +- land explicit issue objects, status/detail wording, and request/retry action + routing +- add bounded health-surface capability exposure and focused automation +- explicitly state which neighboring retained families stay closed + +## Source-backed retained basis + +The queue-head decision is now source-backed rather than README-only. + +Inspected retained donor basis: + +- `C:\visual_studio_solutions\multi_project\GPT 5.4 HyperTwist parse\49-ggml-org-whisper-cpp-upstream-dossier.md` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\stream\stream.cpp` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\command\command.cpp` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\wchess\wchess.cmd\wchess.cmd.cpp` +- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\wchess\wchess.wasm\index-tmpl.html` + +Inspected first-party receiving basis: + +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp` +- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp` + +## Narrowed 6R-V slice decision + +The next widening slice is fixed as: + +1. device-permission and capture-route readiness shell posture + +That narrowed slice covers: + +- first-party speech shell contract expansion for: + - permission contract ids + - capture-route contract ids + - request/retry action ids + - permission/readiness state ids + - route fallback issue objects +- lifecycle sync for: + - open-session posture + - submit-session posture + - close-session posture + - service-health refresh posture +- bounded mock and HTTP speech-health capability exposure for: + - `devicePermissionShell` + - `captureRouteReadinessShell` + - `captureRouteRetry` +- focused automation coverage + +## Deferred neighboring families + +The first `6R-V` implementation packet must keep these capability families +closed: + +- real device-permission workflow ownership +- native audio-device route ownership beyond bounded shell posture +- downloadable model or payload shipping +- provider-profile or BYOK custody +- voice-output ownership +- broad assistant-platform scope + +## Acceptance criteria + +- the packet records why the queue advanced from landed `Phase 6R-U` to + `Phase 6R-V` +- the packet records that the first `6R-V` widening slice is shell posture only +- the packet states exact out-of-scope families for the first `6R-V` pass +- the packet preserves the upstream `MIT` attribution boundary explicitly +- the packet leaves real workflow ownership and payload shipping deferred + +## Validation checklist + +1. confirm the landed `Phase 6R-U` packet is the consumed prior queue head +2. confirm the retained `whisper.cpp` seam narrows cleanly to shell posture + rather than native runtime ownership +3. confirm the next implementation packet is framed as bounded source-backed + widening rather than a broad speech-platform transplant + +That is the packet. + +## Queue effect + +This preparation packet is now consumed by the landed bounded `Phase 6R-V` +implementation slice. + +Once that slice lands, the next clean move should stay narrow inside the +retained `whisper.cpp` row at: + +- a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` downloadable model and + payload custody preparation/control pass diff --git a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md index d76ca37..c735230 100644 --- a/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md +++ b/docs/ops/HYPERTWIST_IMPLEMENTATION_PHASE_1_KICKOFF.md @@ -120,14 +120,18 @@ The next bounded move is now: control pass is now consumed 16. the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell packet is now landed in current code -17. the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` - device-permission and capture-route readiness shell preparation/control pass -18. keep the speech-lane guard visible: +17. the generic source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell control pass is now consumed +18. the bounded permissive `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell packet is now landed in current code +19. the next bounded move is a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` + downloadable model and payload custody preparation/control pass +20. keep the speech-lane guard visible: - keep code-license judgments separate from model, voice, and payload-license review -19. keep the provider-neutral speech-lane guard visible: +21. keep the provider-neutral speech-lane guard visible: - first-party HyperTwist owns the top-level provider/session contract - `whisper.cpp` and `faster-whisper` may win narrower donor slices without inheriting that lane -20. keep the `MagicTile` guard visible: +22. keep the `MagicTile` guard visible: - keep broad non-Euclidean interaction or WinForms/OpenTK host-shell ownership closed by default unless a narrower first-party gap is proven above the landed macro-remapping seam diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md index 4eea805..0ceec1c 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/FEATURE_REGISTRY.md @@ -121,6 +121,7 @@ Do not collapse those three tiers into one undifferentiated "features" voice. | Provider-backed recognition session boundary | Implemented now | first-party current code | `UHyperTwistHttpVisionClient` and normalized recognition envelopes are live first-party seams. | | Speech transcript session boundary | Implemented now | landed `whisper.cpp` `Phase 6R-E` | Transcript sessions, utterance envelopes, and speech-health seams are live. | | Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` | First-party microphone shell profile, bounded panel/action contract, and live shell-state composition are now live above the transcript-session seam. | +| Device-permission and capture-route readiness shell | Implemented now | landed `whisper.cpp` `Phase 6R-V` | First-party permission/readiness shell posture, issue objects, retry/request semantics, and bounded capability exposure are now live above the microphone shell seam. | | Analytics/reporting surfaces | Implemented now | landed analytics/reporting packets | Reporting is real, but bounded to accepted retained slices. | | Browser/spatial/media adjunct surfaces | Implemented now | landed `three.js`, `react-three-fiber`, `xr`, `model-viewer`, `remotion` packets | These are implemented bounded families, not proof of unlimited browser-shell parity. | @@ -199,12 +200,13 @@ repo. |---|---|---|---| | Speech transcript session boundary | Implemented now | landed `whisper.cpp` packet | Current bounded STT truth. | | Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` packet | Current bounded microphone shell truth above the landed transcript-session and orchestration seams. | +| Device-permission and capture-route readiness shell | Implemented now | landed `whisper.cpp` `Phase 6R-V` packet | Current bounded permission/readiness shell posture, route fallback issue objects, and explicit retry/request semantics above the landed microphone shell seam. | | Provider-backed speech session health and transcript envelopes | Implemented now | first-party speech client surfaces | Current first-party speech seam. | | Python transcription-service orchestration profile | Implemented now | landed `faster-whisper` packet | Current bounded Python STT orchestration truth above the existing speech session boundary. | | Local narration sidecar seam | Implemented now | landed `piper` packet | Current bounded offline narration contract, voice catalog, and local sidecar truth. | | Advanced narration orchestration profile | Implemented now | landed `coqui-ai/TTS` packet | Current bounded richer narration profile above the existing companion narration contract. | | Broad voice output / narration sidecars | Deep-source grounded retained | `piper`, `coqui-ai/TTS` retained remainder | Keep downloadable voice/model review separate from the code-license judgment. | -| Device-permission shell and downloadable payload shipping | Deep-source grounded retained | `whisper.cpp` retained remainder | Keep permission workflow, native capture-route readiness, and downloadable model/payload review separate from the landed bounded microphone shell. | +| Downloadable model and payload custody / shipping review | Deep-source grounded retained | `whisper.cpp` retained remainder | Keep model/payload review separate from the code-license judgment and from the landed bounded permission/readiness shell. | ### 8. Provider-neutral AI/provider routing diff --git a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md index 6375386..7d3c883 100644 --- a/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md +++ b/docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md @@ -147,9 +147,13 @@ Canonical discovery surfaces for roadmap interpretation: control pass is now consumed - the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell packet is now landed in current code -- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` - device-permission and capture-route readiness shell preparation/control pass, while keeping - downloadable model/payload review separate from the code-license judgment +- the generic source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell control pass is now consumed +- the bounded permissive `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and + capture-route readiness shell packet is now landed in current code +- the current next bounded move is a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` + downloadable model and payload custody preparation/control pass, while keeping real + device-permission workflow and native capture-route ownership separately deferred - the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than waiting on another first-party packet @@ -211,14 +215,14 @@ Current routing truth: - active non-live implementation-board rows: `34` - retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9` -The next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` -device-permission and capture-route readiness shell preparation/control pass. +The next bounded move is a source-backed `Phase 6R-W` `ggml-org/whisper.cpp` +downloadable model and payload custody preparation/control pass. Queue interpretation after that control pass: - then continue with the retained repo-row queue - highest current repo-row queue pressure sits in: - - the retained `whisper.cpp` device-permission and capture-route readiness shell remainder + - the retained `whisper.cpp` downloadable model and payload custody remainder - `HactarCE/Hyperspeedcube` is now closed for the currently justified retained row: - landed: - puzzle catalog contract @@ -303,13 +307,17 @@ Queue interpretation after that control pass: - transcript segment and transcript result - sidecar health and speech client boundary - live microphone shell profile and shell-state boundary + - device-permission and capture-route readiness shell posture - bounded shell capability exposure for: - microphone capture shell + - device-permission shell + - capture-route readiness shell + - capture-route retry - streaming preview - manual transcript commit - still deferred: - real device-permission workflow - - native capture-route readiness beyond bounded shell state + - native capture-route ownership beyond bounded shell posture - downloadable model or payload shipping - broad assistant-platform scope - `SYSTRAN/faster-whisper` is now landed as a partially incorporated row rather than a generic @@ -356,10 +364,10 @@ Queue interpretation after that control pass: - viseme / gesture runtime integration - broad assistant-platform scope - the next queue shape is now: - - retained `whisper.cpp` device-permission and capture-route readiness shell assessment + - retained `whisper.cpp` downloadable model and payload custody assessment - the next bounded move should stay narrow: - - a source-backed `Phase 6R-V` control pass before any widening into downloadable - model/payload shipping, broad voice-output ownership, or broad assistant-platform scope + - a source-backed `Phase 6R-W` control pass before any widening into real device-permission + workflow, broad voice-output ownership, or broad assistant-platform scope - keep the speech-input / voice sidecar legal sequencing guard visible: - keep code-license judgments separate from model, voice, and payload-license review - keep the provider-neutral speech-lane guard visible: