Implement Phase 6R-U whisper.cpp microphone shell
This commit is contained in:
parent
5ff4937a6e
commit
c56b4500ec
13 changed files with 1261 additions and 28 deletions
|
|
@ -841,6 +841,62 @@ namespace HyperTwistContractLibraryInternal
|
|||
return BrowserShellState;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechMicrophoneShellProfile MakeDefaultCoachCommandMicrophoneShellProfile(const FString& ProfileId)
|
||||
{
|
||||
FHyperTwistSpeechMicrophoneShellProfile Profile;
|
||||
Profile.MicrophoneShellProfileId = ProfileId;
|
||||
Profile.ShellKind = TEXT("microphone-overlay");
|
||||
Profile.CaptureMode = TEXT("vad-gated-short-command");
|
||||
Profile.StepWindowMs = 3000;
|
||||
Profile.CaptureWindowMs = 10000;
|
||||
Profile.KeepWindowMs = 200;
|
||||
Profile.AudioContextTokens = 0;
|
||||
Profile.VadThreshold = 0.6f;
|
||||
Profile.HighPassFrequencyHz = 100.0f;
|
||||
Profile.bKeepContextBetweenChunks = false;
|
||||
Profile.bSupportsStreamingPreview = true;
|
||||
Profile.bSupportsManualCommit = true;
|
||||
|
||||
auto AddPanel = [&Profile](
|
||||
const TCHAR* PanelId,
|
||||
const TCHAR* PanelKind,
|
||||
const TCHAR* AnchorId,
|
||||
const bool bVisibleByDefault
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechShellPanelLayout Panel;
|
||||
Panel.PanelId = PanelId;
|
||||
Panel.PanelKind = PanelKind;
|
||||
Panel.AnchorId = AnchorId;
|
||||
Panel.bVisibleByDefault = bVisibleByDefault;
|
||||
Profile.Panels.Add(Panel);
|
||||
};
|
||||
|
||||
auto AddAction = [&Profile](
|
||||
const TCHAR* ActionId,
|
||||
const TCHAR* InputBinding,
|
||||
const TCHAR* SurfaceId
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechShellActionBinding ActionBinding;
|
||||
ActionBinding.ActionId = ActionId;
|
||||
ActionBinding.InputBinding = InputBinding;
|
||||
ActionBinding.SurfaceId = SurfaceId;
|
||||
Profile.ActionBindings.Add(ActionBinding);
|
||||
};
|
||||
|
||||
AddPanel(TEXT("microphone-status-readout"), TEXT("status-readout"), TEXT("top-left"), true);
|
||||
AddPanel(TEXT("vad-meter"), TEXT("level-meter"), TEXT("left-stack-below-status"), true);
|
||||
AddPanel(TEXT("transcript-preview"), TEXT("transcript-preview"), TEXT("bottom-left"), true);
|
||||
AddPanel(TEXT("command-hints"), TEXT("command-hints"), TEXT("top-right"), true);
|
||||
|
||||
AddAction(TEXT("pause-microphone-capture"), TEXT("P"), TEXT("microphone-status-readout"));
|
||||
AddAction(TEXT("resume-microphone-capture"), TEXT("R"), TEXT("microphone-status-readout"));
|
||||
AddAction(TEXT("commit-transcript"), TEXT("Enter"), TEXT("transcript-preview"));
|
||||
AddAction(TEXT("close-speech-session"), TEXT("Escape"), TEXT("session-root"));
|
||||
return Profile;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCorrectionState MakeClassicCubeCorrectionState(
|
||||
const FHyperTwistVisionSessionConfig& SessionConfig,
|
||||
const FHyperTwistVisionReconstructionSession& ReconstructionSession,
|
||||
|
|
@ -1582,6 +1638,11 @@ FHyperTwistSpeechSessionConfig UHyperTwistContractLibrary::MakeSampleSpeechSessi
|
|||
Config.LanguageMode = TEXT("auto");
|
||||
Config.TaskKind = TEXT("transcribe");
|
||||
Config.GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
Config.MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1");
|
||||
Config.MicrophoneShellProfileDefinition =
|
||||
HyperTwistContractLibraryInternal::MakeDefaultCoachCommandMicrophoneShellProfile(
|
||||
Config.MicrophoneShellProfileId
|
||||
);
|
||||
Config.bEnableVad = true;
|
||||
Config.bEnableWordTimestamps = false;
|
||||
Config.VadPolicy.Threshold = 0.5f;
|
||||
|
|
@ -1662,6 +1723,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistContractLibrary::MakeMockSpeechService
|
|||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints"),
|
||||
TEXT("sessioned-mock-client"),
|
||||
TEXT("microphoneCaptureShell"),
|
||||
TEXT("streamingPreview"),
|
||||
TEXT("manualTranscriptCommit"),
|
||||
TEXT("batchedTranscribe"),
|
||||
TEXT("wordTimestamps"),
|
||||
TEXT("languageDetection"),
|
||||
|
|
|
|||
|
|
@ -317,6 +317,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal
|
|||
TEXT("transcribe"),
|
||||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints"),
|
||||
TEXT("microphoneCaptureShell"),
|
||||
TEXT("streamingPreview"),
|
||||
TEXT("manualTranscriptCommit"),
|
||||
TEXT("batchedTranscribe"),
|
||||
TEXT("wordTimestamps"),
|
||||
TEXT("languageDetection"),
|
||||
|
|
@ -353,6 +356,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal
|
|||
TEXT("transcribe"),
|
||||
TEXT("vadSegments"),
|
||||
TEXT("grammarHints"),
|
||||
TEXT("microphoneCaptureShell"),
|
||||
TEXT("streamingPreview"),
|
||||
TEXT("manualTranscriptCommit"),
|
||||
TEXT("batchedTranscribe"),
|
||||
TEXT("wordTimestamps"),
|
||||
TEXT("languageDetection"),
|
||||
|
|
@ -369,6 +375,10 @@ FHyperTwistSpeechServiceHealth UHyperTwistHttpSpeechClient::GetSpeechServiceHeal
|
|||
{
|
||||
Health.ProviderLabel = ProviderLabel;
|
||||
}
|
||||
Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell"));
|
||||
Health.Capabilities.AddUnique(TEXT("streamingPreview"));
|
||||
Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit"));
|
||||
Health.SupportedOrchestrationProfiles.AddUnique(TEXT("python-batch-transcribe-v1"));
|
||||
if (Health.ServiceEndpoint.IsEmpty())
|
||||
{
|
||||
Health.ServiceEndpoint = ServiceBaseUrl;
|
||||
|
|
|
|||
|
|
@ -128,6 +128,9 @@ FHyperTwistSpeechServiceHealth UHyperTwistMockSpeechClient::GetSpeechServiceHeal
|
|||
Health.ProviderLabel = TEXT("mock");
|
||||
Health.ServiceEndpoint = TEXT("in-process://mock");
|
||||
Health.Capabilities.AddUnique(TEXT("offline-transcribe"));
|
||||
Health.Capabilities.AddUnique(TEXT("microphoneCaptureShell"));
|
||||
Health.Capabilities.AddUnique(TEXT("streamingPreview"));
|
||||
Health.Capabilities.AddUnique(TEXT("manualTranscriptCommit"));
|
||||
Health.Capabilities.AddUnique(TEXT("batchedTranscribe"));
|
||||
Health.Capabilities.AddUnique(TEXT("wordTimestamps"));
|
||||
Health.Capabilities.AddUnique(TEXT("languageDetection"));
|
||||
|
|
|
|||
|
|
@ -2910,6 +2910,156 @@ namespace HyperTwistTrainingSubsystemInternal
|
|||
return BrowserShellState;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechMicrophoneShellProfile BuildDefaultCoachCommandMicrophoneShellProfile(const FString& ProfileId)
|
||||
{
|
||||
FHyperTwistSpeechMicrophoneShellProfile Profile;
|
||||
Profile.MicrophoneShellProfileId = ProfileId;
|
||||
Profile.ShellKind = TEXT("microphone-overlay");
|
||||
Profile.CaptureMode = TEXT("vad-gated-short-command");
|
||||
Profile.StepWindowMs = 3000;
|
||||
Profile.CaptureWindowMs = 10000;
|
||||
Profile.KeepWindowMs = 200;
|
||||
Profile.AudioContextTokens = 0;
|
||||
Profile.VadThreshold = 0.6f;
|
||||
Profile.HighPassFrequencyHz = 100.0f;
|
||||
Profile.bKeepContextBetweenChunks = false;
|
||||
Profile.bSupportsStreamingPreview = true;
|
||||
Profile.bSupportsManualCommit = true;
|
||||
|
||||
auto AddPanel = [&Profile](
|
||||
const TCHAR* PanelId,
|
||||
const TCHAR* PanelKind,
|
||||
const TCHAR* AnchorId,
|
||||
const bool bVisibleByDefault
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechShellPanelLayout Panel;
|
||||
Panel.PanelId = PanelId;
|
||||
Panel.PanelKind = PanelKind;
|
||||
Panel.AnchorId = AnchorId;
|
||||
Panel.bVisibleByDefault = bVisibleByDefault;
|
||||
Profile.Panels.Add(Panel);
|
||||
};
|
||||
|
||||
auto AddAction = [&Profile](
|
||||
const TCHAR* ActionId,
|
||||
const TCHAR* InputBinding,
|
||||
const TCHAR* SurfaceId
|
||||
)
|
||||
{
|
||||
FHyperTwistSpeechShellActionBinding ActionBinding;
|
||||
ActionBinding.ActionId = ActionId;
|
||||
ActionBinding.InputBinding = InputBinding;
|
||||
ActionBinding.SurfaceId = SurfaceId;
|
||||
Profile.ActionBindings.Add(ActionBinding);
|
||||
};
|
||||
|
||||
AddPanel(TEXT("microphone-status-readout"), TEXT("status-readout"), TEXT("top-left"), true);
|
||||
AddPanel(TEXT("vad-meter"), TEXT("level-meter"), TEXT("left-stack-below-status"), true);
|
||||
AddPanel(TEXT("transcript-preview"), TEXT("transcript-preview"), TEXT("bottom-left"), true);
|
||||
AddPanel(TEXT("command-hints"), TEXT("command-hints"), TEXT("top-right"), true);
|
||||
|
||||
AddAction(TEXT("pause-microphone-capture"), TEXT("P"), TEXT("microphone-status-readout"));
|
||||
AddAction(TEXT("resume-microphone-capture"), TEXT("R"), TEXT("microphone-status-readout"));
|
||||
AddAction(TEXT("commit-transcript"), TEXT("Enter"), TEXT("transcript-preview"));
|
||||
AddAction(TEXT("close-speech-session"), TEXT("Escape"), TEXT("session-root"));
|
||||
return Profile;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechMicrophoneShellState BuildCoachCommandMicrophoneShellState(
|
||||
const FHyperTwistSpeechSessionConfig& SessionConfig,
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState* PriorState,
|
||||
const bool bSessionOpen
|
||||
)
|
||||
{
|
||||
const FHyperTwistSpeechMicrophoneShellProfile Profile =
|
||||
SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid()
|
||||
? SessionConfig.MicrophoneShellProfileDefinition
|
||||
: BuildDefaultCoachCommandMicrophoneShellProfile(
|
||||
!SessionConfig.MicrophoneShellProfileId.IsEmpty()
|
||||
? SessionConfig.MicrophoneShellProfileId
|
||||
: TEXT("coach-command-microphone-shell-v1")
|
||||
);
|
||||
|
||||
FHyperTwistSpeechMicrophoneShellState ShellState;
|
||||
ShellState.MicrophoneShellProfileId = Profile.MicrophoneShellProfileId;
|
||||
ShellState.CaptureMode = Profile.CaptureMode;
|
||||
ShellState.ListeningContractId = !SessionConfig.ListeningContractId.IsEmpty()
|
||||
? SessionConfig.ListeningContractId
|
||||
: TEXT("listening-threshold-lifecycle");
|
||||
ShellState.InputRouteId = !SessionConfig.InputRouteId.IsEmpty()
|
||||
? SessionConfig.InputRouteId
|
||||
: TEXT("queue/listening");
|
||||
ShellState.LastLanguageCode =
|
||||
SessionConfig.LanguageMode.Equals(TEXT("auto"), ESearchCase::IgnoreCase)
|
||||
? TEXT("en")
|
||||
: SessionConfig.LanguageMode;
|
||||
ShellState.StepWindowMs = Profile.StepWindowMs;
|
||||
ShellState.CaptureWindowMs = Profile.CaptureWindowMs;
|
||||
ShellState.KeepWindowMs = Profile.KeepWindowMs;
|
||||
ShellState.VadThreshold = Profile.VadThreshold;
|
||||
ShellState.bSessionOpen = bSessionOpen;
|
||||
ShellState.bMicrophonePermissionGranted = true;
|
||||
ShellState.bCaptureRouteReady = true;
|
||||
ShellState.bCaptureActive = bSessionOpen;
|
||||
ShellState.bVadArmed = SessionConfig.bEnableVad;
|
||||
ShellState.bAwaitingSpeech = bSessionOpen;
|
||||
ShellState.bTranscriptionInFlight = false;
|
||||
ShellState.bCommitReady = false;
|
||||
ShellState.bPermissionPromptVisible = false;
|
||||
|
||||
for (const FHyperTwistSpeechShellPanelLayout& Panel : Profile.Panels)
|
||||
{
|
||||
if (Panel.PanelId == TEXT("microphone-status-readout"))
|
||||
{
|
||||
ShellState.bStatusPanelVisible = Panel.bVisibleByDefault;
|
||||
}
|
||||
else if (Panel.PanelId == TEXT("vad-meter"))
|
||||
{
|
||||
ShellState.bVadMeterVisible = Panel.bVisibleByDefault;
|
||||
}
|
||||
else if (Panel.PanelId == TEXT("transcript-preview"))
|
||||
{
|
||||
ShellState.bTranscriptPreviewVisible = Panel.bVisibleByDefault;
|
||||
}
|
||||
else if (Panel.PanelId == TEXT("command-hints"))
|
||||
{
|
||||
ShellState.bCommandHintsVisible = Panel.bVisibleByDefault;
|
||||
}
|
||||
}
|
||||
|
||||
for (const FHyperTwistSpeechShellActionBinding& ActionBinding : Profile.ActionBindings)
|
||||
{
|
||||
ShellState.AvailableActionIds.Add(ActionBinding.ActionId);
|
||||
}
|
||||
|
||||
if (PriorState != nullptr)
|
||||
{
|
||||
ShellState.SubmittedUtteranceCount = FMath::Max(0, PriorState->SubmittedUtteranceCount);
|
||||
ShellState.FinalTranscriptCount = FMath::Max(0, PriorState->FinalTranscriptCount);
|
||||
ShellState.bCommitReady =
|
||||
PriorState->bHasTranscriptResult && PriorState->LastTranscriptResult.IsStructurallyValid();
|
||||
ShellState.LastCommittedUtteranceId =
|
||||
!PriorState->LastTranscriptResult.UtteranceId.IsEmpty()
|
||||
? PriorState->LastTranscriptResult.UtteranceId
|
||||
: FString();
|
||||
ShellState.LastTranscriptPreview = PriorState->LastTranscriptResult.TranscriptText;
|
||||
if (!PriorState->LastTranscriptResult.LanguageCode.IsEmpty())
|
||||
{
|
||||
ShellState.LastLanguageCode = PriorState->LastTranscriptResult.LanguageCode;
|
||||
}
|
||||
|
||||
if (PriorState->MicrophoneShellState.IsStructurallyValid())
|
||||
{
|
||||
ShellState.DetectedSpeechStartMs = PriorState->MicrophoneShellState.DetectedSpeechStartMs;
|
||||
ShellState.DetectedSpeechEndMs = PriorState->MicrophoneShellState.DetectedSpeechEndMs;
|
||||
ShellState.LastSilenceGapMs = PriorState->MicrophoneShellState.LastSilenceGapMs;
|
||||
}
|
||||
}
|
||||
|
||||
return ShellState;
|
||||
}
|
||||
|
||||
FHyperTwistVisionCorrectionState BuildClassicCubeCorrectionState(
|
||||
const FHyperTwistVisionSessionConfig& SessionConfig,
|
||||
const FHyperTwistVisionReconstructionSession& ReconstructionSession,
|
||||
|
|
@ -8260,7 +8410,15 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out
|
|||
if (ActiveCompanionSpeechSessionState.bSessionOpen
|
||||
&& ActiveCompanionSpeechSessionState.ActiveSessionId == SessionConfig.SessionId)
|
||||
{
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState PriorSessionState =
|
||||
ActiveCompanionSpeechSessionState;
|
||||
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState =
|
||||
HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState(
|
||||
SessionConfig,
|
||||
&PriorSessionState,
|
||||
true
|
||||
);
|
||||
ActiveCompanionSpeechSessionState.LastError.Reset();
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError.Reset();
|
||||
return true;
|
||||
|
|
@ -8292,6 +8450,12 @@ bool UHyperTwistTrainingSubsystem::OpenActiveCompanionSpeechSession(FString& Out
|
|||
ActiveCompanionSpeechSessionState.SessionConfig = SessionConfig;
|
||||
ActiveCompanionSpeechSessionState.bHasTranscriptResult = false;
|
||||
ActiveCompanionSpeechSessionState.LastTranscriptResult = FHyperTwistSpeechTranscriptResult();
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState =
|
||||
HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState(
|
||||
SessionConfig,
|
||||
&ActiveCompanionSpeechSessionState,
|
||||
true
|
||||
);
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
|
@ -8361,6 +8525,30 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp
|
|||
NormalizedUtterance.SpeechStartMs,
|
||||
NormalizedUtterance.SpeechEndMs
|
||||
);
|
||||
if (!ActiveCompanionSpeechSessionState.MicrophoneShellState.IsStructurallyValid())
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState =
|
||||
HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState(
|
||||
ActiveCompanionSpeechSessionState.SessionConfig,
|
||||
&ActiveCompanionSpeechSessionState,
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId = NormalizedUtterance.UtteranceId;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.DetectedSpeechStartMs =
|
||||
NormalizedUtterance.SpeechStartMs;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.DetectedSpeechEndMs =
|
||||
NormalizedUtterance.SpeechEndMs;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.LastSilenceGapMs =
|
||||
NormalizedUtterance.SilenceGapMs;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bSessionOpen = true;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = false;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bVadArmed =
|
||||
ActiveCompanionSpeechSessionState.SessionConfig.bEnableVad;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = false;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = true;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bCommitReady = false;
|
||||
|
||||
Result = SpeechClient->TranscribeSpeechUtterance(NormalizedUtterance);
|
||||
if (Result.SessionId.IsEmpty())
|
||||
|
|
@ -8449,6 +8637,28 @@ FHyperTwistSpeechTranscriptResult UHyperTwistTrainingSubsystem::SubmitActiveComp
|
|||
ActiveCompanionSpeechSessionState.LastError =
|
||||
Result.Warnings.Num() > 0 ? FString::Join(Result.Warnings, TEXT(" | ")) : FString();
|
||||
ActiveCompanionSpeechSessionState.ServiceHealth.LastError = ActiveCompanionSpeechSessionState.LastError;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId.Reset();
|
||||
if (Result.bIsFinal && Result.IsStructurallyValid())
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.LastCommittedUtteranceId = Result.UtteranceId;
|
||||
}
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.LastTranscriptPreview = Result.TranscriptText;
|
||||
if (!Result.LanguageCode.IsEmpty())
|
||||
{
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.LastLanguageCode = Result.LanguageCode;
|
||||
}
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.SubmittedUtteranceCount =
|
||||
ActiveCompanionSpeechSessionState.SubmittedUtteranceCount;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.FinalTranscriptCount =
|
||||
ActiveCompanionSpeechSessionState.FinalTranscriptCount;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bSessionOpen = true;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = true;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bVadArmed =
|
||||
ActiveCompanionSpeechSessionState.SessionConfig.bEnableVad;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = true;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = false;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bCommitReady =
|
||||
Result.bIsFinal && Result.IsStructurallyValid();
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
|
@ -8487,6 +8697,16 @@ bool UHyperTwistTrainingSubsystem::CloseActiveCompanionSpeechSession(FString& Ou
|
|||
ActiveCompanionSpeechSessionState.ActiveSessionId.Reset();
|
||||
ActiveCompanionSpeechSessionState.LastUpdatedAtUtc = FDateTime::UtcNow().ToIso8601();
|
||||
ActiveCompanionSpeechSessionState.LastError.Reset();
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState =
|
||||
HyperTwistTrainingSubsystemInternal::BuildCoachCommandMicrophoneShellState(
|
||||
ActiveCompanionSpeechSessionState.SessionConfig,
|
||||
&ActiveCompanionSpeechSessionState,
|
||||
false
|
||||
);
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.ActiveUtteranceId.Reset();
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bCaptureActive = false;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bAwaitingSpeech = false;
|
||||
ActiveCompanionSpeechSessionState.MicrophoneShellState.bTranscriptionInFlight = false;
|
||||
RefreshCompanionSpeechServiceHealth();
|
||||
return true;
|
||||
}
|
||||
|
|
@ -8989,6 +9209,28 @@ FHyperTwistSpeechSessionConfig UHyperTwistTrainingSubsystem::BuildActiveCompanio
|
|||
{
|
||||
SessionConfig.GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
}
|
||||
if (SessionConfig.MicrophoneShellProfileId.IsEmpty()
|
||||
&& SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid())
|
||||
{
|
||||
SessionConfig.MicrophoneShellProfileId =
|
||||
SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId;
|
||||
}
|
||||
if (SessionConfig.MicrophoneShellProfileId.IsEmpty())
|
||||
{
|
||||
SessionConfig.MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1");
|
||||
}
|
||||
if (!SessionConfig.MicrophoneShellProfileDefinition.IsStructurallyValid())
|
||||
{
|
||||
SessionConfig.MicrophoneShellProfileDefinition =
|
||||
HyperTwistTrainingSubsystemInternal::BuildDefaultCoachCommandMicrophoneShellProfile(
|
||||
SessionConfig.MicrophoneShellProfileId
|
||||
);
|
||||
}
|
||||
if (SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId.IsEmpty())
|
||||
{
|
||||
SessionConfig.MicrophoneShellProfileDefinition.MicrophoneShellProfileId =
|
||||
SessionConfig.MicrophoneShellProfileId;
|
||||
}
|
||||
SessionConfig.bEnableVad = true;
|
||||
if (!SessionConfig.VadPolicy.IsStructurallyValid())
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1434,6 +1434,270 @@ struct FHyperTwistSpeechTranscriptionOrchestrationProfile
|
|||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechShellPanelLayout
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString PanelId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString PanelKind;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString AnchorId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bVisibleByDefault = true;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return !PanelId.IsEmpty()
|
||||
&& !PanelKind.IsEmpty()
|
||||
&& !AnchorId.IsEmpty();
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechShellActionBinding
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ActionId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString InputBinding;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString SurfaceId;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
return !ActionId.IsEmpty()
|
||||
&& !InputBinding.IsEmpty()
|
||||
&& !SurfaceId.IsEmpty();
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechMicrophoneShellProfile
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString MicrophoneShellProfileId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ShellKind = TEXT("microphone-overlay");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString CaptureMode = TEXT("vad-gated-short-command");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 StepWindowMs = 3000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 CaptureWindowMs = 10000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 KeepWindowMs = 200;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 AudioContextTokens = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float VadThreshold = 0.6f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float HighPassFrequencyHz = 100.0f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bKeepContextBetweenChunks = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSupportsStreamingPreview = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSupportsManualCommit = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FHyperTwistSpeechShellPanelLayout> Panels;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FHyperTwistSpeechShellActionBinding> ActionBindings;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
if (MicrophoneShellProfileId.IsEmpty()
|
||||
|| ShellKind.IsEmpty()
|
||||
|| CaptureMode.IsEmpty()
|
||||
|| StepWindowMs <= 0
|
||||
|| CaptureWindowMs < StepWindowMs
|
||||
|| KeepWindowMs < 0
|
||||
|| KeepWindowMs > CaptureWindowMs
|
||||
|| AudioContextTokens < 0
|
||||
|| VadThreshold <= 0.0f
|
||||
|| HighPassFrequencyHz < 0.0f
|
||||
|| Panels.Num() <= 0
|
||||
|| ActionBindings.Num() <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const FHyperTwistSpeechShellPanelLayout& Panel : Panels)
|
||||
{
|
||||
if (!Panel.IsStructurallyValid())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
for (const FHyperTwistSpeechShellActionBinding& ActionBinding : ActionBindings)
|
||||
{
|
||||
if (!ActionBinding.IsStructurallyValid())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechMicrophoneShellState
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString MicrophoneShellProfileId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString CaptureMode = TEXT("vad-gated-short-command");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ListeningContractId = TEXT("listening-threshold-lifecycle");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString InputRouteId = TEXT("queue/listening");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString ActiveUtteranceId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastCommittedUtteranceId;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastTranscriptPreview;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString LastLanguageCode = TEXT("en");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 StepWindowMs = 3000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 CaptureWindowMs = 10000;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 KeepWindowMs = 200;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 SubmittedUtteranceCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 FinalTranscriptCount = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 DetectedSpeechStartMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 DetectedSpeechEndMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
int32 LastSilenceGapMs = 0;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
float VadThreshold = 0.6f;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bSessionOpen = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bMicrophonePermissionGranted = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bCaptureRouteReady = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bCaptureActive = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bVadArmed = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bAwaitingSpeech = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bTranscriptionInFlight = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bCommitReady = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bStatusPanelVisible = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bVadMeterVisible = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bTranscriptPreviewVisible = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bCommandHintsVisible = true;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bPermissionPromptVisible = false;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
TArray<FString> AvailableActionIds;
|
||||
|
||||
bool IsStructurallyValid() const
|
||||
{
|
||||
if (MicrophoneShellProfileId.IsEmpty()
|
||||
|| CaptureMode.IsEmpty()
|
||||
|| ListeningContractId.IsEmpty()
|
||||
|| InputRouteId.IsEmpty()
|
||||
|| LastLanguageCode.IsEmpty()
|
||||
|| StepWindowMs <= 0
|
||||
|| CaptureWindowMs < StepWindowMs
|
||||
|| KeepWindowMs < 0
|
||||
|| KeepWindowMs > CaptureWindowMs
|
||||
|| SubmittedUtteranceCount < 0
|
||||
|| FinalTranscriptCount < 0
|
||||
|| DetectedSpeechStartMs < 0
|
||||
|| DetectedSpeechEndMs < DetectedSpeechStartMs
|
||||
|| LastSilenceGapMs < 0
|
||||
|| VadThreshold <= 0.0f
|
||||
|| AvailableActionIds.Num() <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const FString& ActionId : AvailableActionIds)
|
||||
{
|
||||
if (ActionId.IsEmpty())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
USTRUCT(BlueprintType)
|
||||
struct FHyperTwistSpeechSessionConfig
|
||||
{
|
||||
|
|
@ -1466,6 +1730,12 @@ struct FHyperTwistSpeechSessionConfig
|
|||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString GrammarProfileId = TEXT("coach-command-grammar-v1");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FString MicrophoneShellProfileId = TEXT("coach-command-microphone-shell-v1");
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechMicrophoneShellProfile MicrophoneShellProfileDefinition;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
bool bEnableVad = true;
|
||||
|
||||
|
|
@ -1490,6 +1760,8 @@ struct FHyperTwistSpeechSessionConfig
|
|||
&& SampleRateHz > 0
|
||||
&& ChannelCount > 0
|
||||
&& !TaskKind.IsEmpty()
|
||||
&& !MicrophoneShellProfileId.IsEmpty()
|
||||
&& MicrophoneShellProfileDefinition.IsStructurallyValid()
|
||||
&& VadPolicy.IsStructurallyValid()
|
||||
&& OrchestrationProfile.IsStructurallyValid();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1937,6 +1937,9 @@ struct FHyperTwistTrainingCompanionSpeechSessionState
|
|||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechServiceHealth ServiceHealth;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechMicrophoneShellState MicrophoneShellState;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
|
||||
FHyperTwistSpeechTranscriptResult LastTranscriptResult;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,278 @@
|
|||
// Copyright HyperTwist, Inc. All Rights Reserved.
|
||||
|
||||
#include "Misc/AutomationTest.h"
|
||||
|
||||
#include "Engine/GameInstance.h"
|
||||
#include "HyperTwistTraining/HyperTwistTrainingSubsystem.h"
|
||||
#include "UObject/UnrealType.h"
|
||||
|
||||
#if WITH_AUTOMATION_TESTS
|
||||
|
||||
namespace HyperTwistWhisperCppPhase6RUTestInternal
|
||||
{
|
||||
FHyperTwistTrainingDeck MakeSpeechDeck()
|
||||
{
|
||||
FHyperTwistTrainingDeck Deck;
|
||||
Deck.DeckId = TEXT("phase6r-u/whispercpp-microphone-shell");
|
||||
Deck.Title = TEXT("Phase 6R-U Whisper Microphone Shell");
|
||||
Deck.DeliveryModes = {
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
};
|
||||
|
||||
FHyperTwistTrainingCase TrainingCase;
|
||||
TrainingCase.CaseId = TEXT("phase6r-u-case");
|
||||
TrainingCase.PuzzleId = TEXT("cube/3x3x3");
|
||||
TrainingCase.PromptKind = EHyperTwistTrainingPromptKind::Sequence;
|
||||
TrainingCase.PromptLabel = TEXT("Phase 6R-U Coach Speech");
|
||||
TrainingCase.AllowedDeliveryModes = {
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
};
|
||||
Deck.Cases = {TrainingCase};
|
||||
return Deck;
|
||||
}
|
||||
|
||||
void ForceMockSpeechClient(UHyperTwistTrainingSubsystem* TrainingSubsystem)
|
||||
{
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (FStrProperty* SpeechClientKindProperty = FindFProperty<FStrProperty>(
|
||||
UHyperTwistTrainingSubsystem::StaticClass(),
|
||||
TEXT("CompanionSpeechClientKind")
|
||||
))
|
||||
{
|
||||
SpeechClientKindProperty->SetPropertyValue_InContainer(TrainingSubsystem, TEXT("mock"));
|
||||
}
|
||||
}
|
||||
|
||||
UHyperTwistTrainingSubsystem* MakeSpeechSubsystem(const FString& SessionId)
|
||||
{
|
||||
UGameInstance* GameInstance = NewObject<UGameInstance>(GetTransientPackage());
|
||||
if (GameInstance == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem = NewObject<UHyperTwistTrainingSubsystem>(GameInstance);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ForceMockSpeechClient(TrainingSubsystem);
|
||||
const FHyperTwistTrainingRunState RunState = TrainingSubsystem->StartTrainingRunFromDeck(
|
||||
MakeSpeechDeck(),
|
||||
TEXT("phase6r-u-user"),
|
||||
SessionId,
|
||||
EHyperTwistTrainingDeliveryMode::CoachReviewed
|
||||
);
|
||||
return RunState.IsStructurallyValid() ? TrainingSubsystem : nullptr;
|
||||
}
|
||||
|
||||
const FHyperTwistSpeechShellPanelLayout* FindPanel(
|
||||
const FHyperTwistSpeechMicrophoneShellProfile& Profile,
|
||||
const FString& PanelId
|
||||
)
|
||||
{
|
||||
return Profile.Panels.FindByPredicate(
|
||||
[&PanelId](const FHyperTwistSpeechShellPanelLayout& Candidate)
|
||||
{
|
||||
return Candidate.PanelId == PanelId;
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
const FHyperTwistSpeechShellActionBinding* FindAction(
|
||||
const FHyperTwistSpeechMicrophoneShellProfile& Profile,
|
||||
const FString& ActionId
|
||||
)
|
||||
{
|
||||
return Profile.ActionBindings.FindByPredicate(
|
||||
[&ActionId](const FHyperTwistSpeechShellActionBinding& Candidate)
|
||||
{
|
||||
return Candidate.ActionId == ActionId;
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RUMicrophoneShellSessionConfigTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellSessionConfig",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RUMicrophoneShellSessionConfigTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-config-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
TestTrue(TEXT("The microphone shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError));
|
||||
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
const FHyperTwistSpeechMicrophoneShellProfile& Profile =
|
||||
SessionState.SessionConfig.MicrophoneShellProfileDefinition;
|
||||
|
||||
TestEqual(
|
||||
TEXT("The microphone shell route must preserve the first-party shell profile id."),
|
||||
SessionState.SessionConfig.MicrophoneShellProfileId,
|
||||
TEXT("coach-command-microphone-shell-v1")
|
||||
);
|
||||
TestTrue(TEXT("The microphone shell route must expose a structurally valid shell profile."), Profile.IsStructurallyValid());
|
||||
TestEqual(TEXT("The microphone shell must retain four bounded panels."), Profile.Panels.Num(), 4);
|
||||
TestEqual(TEXT("The microphone shell must retain four bounded actions."), Profile.ActionBindings.Num(), 4);
|
||||
TestEqual(TEXT("The microphone shell must preserve the donor-backed step window."), Profile.StepWindowMs, 3000);
|
||||
TestEqual(TEXT("The microphone shell must preserve the donor-backed capture window."), Profile.CaptureWindowMs, 10000);
|
||||
TestEqual(TEXT("The microphone shell must preserve the donor-backed keep window."), Profile.KeepWindowMs, 200);
|
||||
TestEqual(TEXT("The microphone shell must preserve zero audio-context tokens in the bounded route."), Profile.AudioContextTokens, 0);
|
||||
TestTrue(TEXT("The microphone shell must preserve the donor-backed VAD threshold."), FMath::IsNearlyEqual(Profile.VadThreshold, 0.6f));
|
||||
TestTrue(TEXT("The microphone shell must preserve the donor-backed high-pass floor."), FMath::IsNearlyEqual(Profile.HighPassFrequencyHz, 100.0f));
|
||||
TestFalse(TEXT("The microphone shell must keep previous-text carryover disabled in the bounded route."), Profile.bKeepContextBetweenChunks);
|
||||
TestTrue(TEXT("The microphone shell must advertise streaming preview."), Profile.bSupportsStreamingPreview);
|
||||
TestTrue(TEXT("The microphone shell must advertise manual commit."), Profile.bSupportsManualCommit);
|
||||
|
||||
const FHyperTwistSpeechShellPanelLayout* VadPanel =
|
||||
HyperTwistWhisperCppPhase6RUTestInternal::FindPanel(Profile, TEXT("vad-meter"));
|
||||
TestNotNull(TEXT("The VAD meter panel must be present."), VadPanel);
|
||||
if (VadPanel != nullptr)
|
||||
{
|
||||
TestEqual(TEXT("The VAD meter must stay anchored below the status readout."), VadPanel->AnchorId, TEXT("left-stack-below-status"));
|
||||
}
|
||||
|
||||
const FHyperTwistSpeechShellActionBinding* CommitAction =
|
||||
HyperTwistWhisperCppPhase6RUTestInternal::FindAction(Profile, TEXT("commit-transcript"));
|
||||
TestNotNull(TEXT("The manual transcript commit action must be present."), CommitAction);
|
||||
if (CommitAction != nullptr)
|
||||
{
|
||||
TestEqual(TEXT("The manual transcript commit action must keep the Enter binding."), CommitAction->InputBinding, TEXT("Enter"));
|
||||
}
|
||||
|
||||
TestTrue(
|
||||
TEXT("The speech service health must expose microphone-shell capability."),
|
||||
SessionState.ServiceHealth.Capabilities.Contains(TEXT("microphoneCaptureShell"))
|
||||
);
|
||||
TestTrue(
|
||||
TEXT("The speech service health must expose streaming-preview capability."),
|
||||
SessionState.ServiceHealth.Capabilities.Contains(TEXT("streamingPreview"))
|
||||
);
|
||||
TestTrue(
|
||||
TEXT("The speech service health must expose manual transcript commit capability."),
|
||||
SessionState.ServiceHealth.Capabilities.Contains(TEXT("manualTranscriptCommit"))
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RUMicrophoneShellOpenStateTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellOpenState",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RUMicrophoneShellOpenStateTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-open-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FString OpenError;
|
||||
TestTrue(TEXT("The microphone shell route must open a speech session."), TrainingSubsystem->OpenActiveCompanionSpeechSession(OpenError));
|
||||
TestTrue(TEXT("Opening the speech session must not report an error."), OpenError.IsEmpty());
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState;
|
||||
|
||||
TestTrue(TEXT("The microphone shell open state must be structurally valid."), ShellState.IsStructurallyValid());
|
||||
TestEqual(TEXT("The microphone shell state must preserve the first-party profile id."), ShellState.MicrophoneShellProfileId, TEXT("coach-command-microphone-shell-v1"));
|
||||
TestEqual(TEXT("The microphone shell state must preserve the listening contract."), ShellState.ListeningContractId, TEXT("listening-threshold-lifecycle"));
|
||||
TestEqual(TEXT("The microphone shell state must preserve the input route."), ShellState.InputRouteId, TEXT("queue/listening"));
|
||||
TestEqual(TEXT("The microphone shell state must preserve the donor-backed step window."), ShellState.StepWindowMs, 3000);
|
||||
TestEqual(TEXT("The microphone shell state must preserve the donor-backed capture window."), ShellState.CaptureWindowMs, 10000);
|
||||
TestEqual(TEXT("The microphone shell state must preserve the donor-backed keep window."), ShellState.KeepWindowMs, 200);
|
||||
TestTrue(TEXT("The microphone shell must report the session open."), ShellState.bSessionOpen);
|
||||
TestTrue(TEXT("The microphone shell must report the capture route ready."), ShellState.bCaptureRouteReady);
|
||||
TestTrue(TEXT("The microphone shell must start capture-active."), ShellState.bCaptureActive);
|
||||
TestTrue(TEXT("The microphone shell must start with VAD armed."), ShellState.bVadArmed);
|
||||
TestTrue(TEXT("The microphone shell must start awaiting speech."), ShellState.bAwaitingSpeech);
|
||||
TestFalse(TEXT("The microphone shell must not start with transcription in flight."), ShellState.bTranscriptionInFlight);
|
||||
TestFalse(TEXT("The microphone shell must not start commit-ready."), ShellState.bCommitReady);
|
||||
TestTrue(TEXT("The microphone shell must keep the status panel visible."), ShellState.bStatusPanelVisible);
|
||||
TestTrue(TEXT("The microphone shell must keep the VAD meter visible."), ShellState.bVadMeterVisible);
|
||||
TestTrue(TEXT("The microphone shell must keep the transcript preview visible."), ShellState.bTranscriptPreviewVisible);
|
||||
TestTrue(TEXT("The microphone shell must keep the command hints visible."), ShellState.bCommandHintsVisible);
|
||||
TestFalse(TEXT("The microphone shell must keep the permission prompt hidden."), ShellState.bPermissionPromptVisible);
|
||||
TestEqual(TEXT("The microphone shell must start with no submitted utterances."), ShellState.SubmittedUtteranceCount, 0);
|
||||
TestEqual(TEXT("The microphone shell must start with no final transcripts."), ShellState.FinalTranscriptCount, 0);
|
||||
TestEqual(TEXT("The microphone shell must expose four bounded shell actions."), ShellState.AvailableActionIds.Num(), 4);
|
||||
TestTrue(TEXT("The microphone shell must expose pause capture."), ShellState.AvailableActionIds.Contains(TEXT("pause-microphone-capture")));
|
||||
TestTrue(TEXT("The microphone shell must expose resume capture."), ShellState.AvailableActionIds.Contains(TEXT("resume-microphone-capture")));
|
||||
TestTrue(TEXT("The microphone shell must expose manual transcript commit."), ShellState.AvailableActionIds.Contains(TEXT("commit-transcript")));
|
||||
TestTrue(TEXT("The microphone shell must expose session close."), ShellState.AvailableActionIds.Contains(TEXT("close-speech-session")));
|
||||
return true;
|
||||
}
|
||||
|
||||
IMPLEMENT_SIMPLE_AUTOMATION_TEST(
|
||||
FHyperTwistWhisperCppPhase6RUMicrophoneShellUtteranceStateTest,
|
||||
"HyperTwist.Permissive.WhisperCpp.Phase6R.U.MicrophoneShellUtteranceState",
|
||||
EAutomationTestFlags::EditorContext | EAutomationTestFlags::EngineFilter
|
||||
)
|
||||
|
||||
bool FHyperTwistWhisperCppPhase6RUMicrophoneShellUtteranceStateTest::RunTest(const FString& Parameters)
|
||||
{
|
||||
UHyperTwistTrainingSubsystem* TrainingSubsystem =
|
||||
HyperTwistWhisperCppPhase6RUTestInternal::MakeSpeechSubsystem(TEXT("phase6r-u-utterance-session"));
|
||||
TestNotNull(TEXT("The speech subsystem must be constructed for Phase 6R-U."), TrainingSubsystem);
|
||||
if (TrainingSubsystem == nullptr)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
FHyperTwistSpeechUtteranceEnvelope Utterance;
|
||||
Utterance.AudioRef = TEXT("mock://coach/next-case");
|
||||
Utterance.SpeechStartMs = 140;
|
||||
Utterance.SpeechEndMs = 980;
|
||||
Utterance.SilenceGapMs = 240;
|
||||
Utterance.SampleCount = 13440;
|
||||
const FHyperTwistSpeechTranscriptResult Result =
|
||||
TrainingSubsystem->SubmitActiveCompanionSpeechUtterance(Utterance);
|
||||
|
||||
TestTrue(TEXT("The microphone shell utterance route must return a structurally valid transcript result."), Result.IsStructurallyValid());
|
||||
TestEqual(TEXT("The bounded mock route must resolve the next-case transcript."), Result.TranscriptText, TEXT("next case"));
|
||||
|
||||
const FHyperTwistTrainingCompanionSpeechSessionState SessionState =
|
||||
TrainingSubsystem->GetActiveCompanionSpeechSessionState();
|
||||
const FHyperTwistSpeechMicrophoneShellState& ShellState = SessionState.MicrophoneShellState;
|
||||
|
||||
TestTrue(TEXT("The microphone shell utterance state must be structurally valid."), ShellState.IsStructurallyValid());
|
||||
TestTrue(TEXT("The microphone shell must keep the speech session open after a final utterance."), ShellState.bSessionOpen);
|
||||
TestTrue(TEXT("The microphone shell must resume capture after a final utterance."), ShellState.bCaptureActive);
|
||||
TestTrue(TEXT("The microphone shell must resume awaiting speech after a final utterance."), ShellState.bAwaitingSpeech);
|
||||
TestFalse(TEXT("The microphone shell must clear transcription-in-flight after a final utterance."), ShellState.bTranscriptionInFlight);
|
||||
TestTrue(TEXT("The microphone shell must become commit-ready after a final utterance."), ShellState.bCommitReady);
|
||||
TestTrue(TEXT("The microphone shell must clear the active utterance id after a final utterance."), ShellState.ActiveUtteranceId.IsEmpty());
|
||||
TestEqual(TEXT("The microphone shell must retain the last committed utterance id."), ShellState.LastCommittedUtteranceId, Result.UtteranceId);
|
||||
TestEqual(TEXT("The microphone shell must retain the last transcript preview."), ShellState.LastTranscriptPreview, TEXT("next case"));
|
||||
TestEqual(TEXT("The microphone shell must retain the transcript language."), ShellState.LastLanguageCode, TEXT("en"));
|
||||
TestEqual(TEXT("The microphone shell must count one submitted utterance."), ShellState.SubmittedUtteranceCount, 1);
|
||||
TestEqual(TEXT("The microphone shell must count one final transcript."), ShellState.FinalTranscriptCount, 1);
|
||||
TestEqual(TEXT("The microphone shell must retain the detected speech start."), ShellState.DetectedSpeechStartMs, 140);
|
||||
TestEqual(TEXT("The microphone shell must retain the detected speech end."), ShellState.DetectedSpeechEndMs, 980);
|
||||
TestEqual(TEXT("The microphone shell must retain the last silence gap."), ShellState.LastSilenceGapMs, 240);
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -167,9 +167,13 @@ Status update on `2026-05-21`:
|
|||
control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping packet
|
||||
is now landed in current code
|
||||
- the current next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp`
|
||||
live microphone capture shell preparation/control pass, not a new restrictive packet by
|
||||
default
|
||||
- the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell
|
||||
control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell
|
||||
packet is now landed in current code
|
||||
- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp`
|
||||
device-permission and capture-route readiness shell preparation/control pass, not a new
|
||||
restrictive packet by default
|
||||
- use the repo-row README census, the portfolio standing refresh backfill, and the `2R-A`
|
||||
ownership contract for the current queue after that correction
|
||||
|
||||
|
|
@ -273,8 +277,8 @@ Current practical interpretation:
|
|||
- the landed `PostHog/posthog` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_D_POSTHOG_CONTROL_PLANE_IMPLEMENTATION_2026-05-13.md`
|
||||
- the landed `screenpipe/screenpipe` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_E_SCREENPIPE_CAPTURE_HISTORY_IMPLEMENTATION_2026-05-13.md`
|
||||
- the landed `remotion-dev/remotion` boundary-sensitive widening packet now lives in `docs/HYPERTWIST_PHASE_4R_PACKET_4R_F_REMOTION_MEDIA_EXPORT_IMPLEMENTATION_2026-05-13.md`
|
||||
- the next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp`
|
||||
live microphone capture shell preparation/control pass
|
||||
- the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp`
|
||||
device-permission and capture-route readiness shell preparation/control pass
|
||||
|
||||
Companion docs:
|
||||
|
||||
|
|
@ -1522,9 +1526,12 @@ Approved working posture:
|
|||
- keep it bounded behind a speech-input seam
|
||||
- prefer command, dictation, and constrained coach-interaction use over broad “voice assistant platform” scope
|
||||
- the first bounded permissive implementation slice is now landed as a speech transcript session boundary above the existing companion listening lifecycle
|
||||
- the next bounded permissive implementation slice is now landed as a live microphone shell
|
||||
profile and shell-state boundary above the existing transcript-session seam
|
||||
- the top-level provider/session contract remains first-party HyperTwist-owned and
|
||||
provider-neutral; this row does not own that lane
|
||||
- keep model files, downloadable payloads, and future voice-asset review separate from the code-license judgment
|
||||
- keep real device-permission workflow, model files, downloadable payloads, and future
|
||||
voice-asset review separate from the code-license judgment
|
||||
- treat `SYSTRAN/faster-whisper` as the landed complementary Python-orchestration donor rather
|
||||
than widening `whisper.cpp` into the runtime core
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,160 @@
|
|||
# HyperTwist Phase 6R-U whisper.cpp live microphone capture shell implementation packet
|
||||
|
||||
Created on `2026-05-24`
|
||||
|
||||
## Status
|
||||
|
||||
- first-party HyperTwist packet
|
||||
- bounded permissive `Phase 6R-U` implementation slice
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet lands the next narrower bounded slice from the retained
|
||||
`ggml-org/whisper.cpp` speech-input row.
|
||||
|
||||
The landed slice is:
|
||||
|
||||
- first-party live microphone capture shell profile and session-state boundary
|
||||
above the existing companion speech transcript-session seam
|
||||
|
||||
It is not:
|
||||
|
||||
- a full `whisper.cpp` row transplant
|
||||
- a real device-permission workflow packet
|
||||
- a downloadable model or payload-shipping packet
|
||||
- a top-level provider/session contract packet
|
||||
- a voice-output packet
|
||||
- a broad assistant-platform packet
|
||||
|
||||
## Current authority basis
|
||||
|
||||
This implementation packet stands on:
|
||||
|
||||
- `docs/REPO_LICENSE_TRACKING.md`
|
||||
- `docs/ops/HYPERTWIST_CROSS_LANE_AUTHORITY_HIERARCHY_AND_RECONCILIATION_2026-05-20.md`
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_PREPARATION_PACKET_2026-05-24.md`
|
||||
|
||||
The retained owner remains:
|
||||
|
||||
- `ggml-org/whisper.cpp`
|
||||
|
||||
The top-level provider/session contract owner does not change here:
|
||||
|
||||
- first-party HyperTwist remains the provider-neutral owner for the speech
|
||||
provider/session lane
|
||||
- `SYSTRAN/faster-whisper` remains the complementary landed Python
|
||||
orchestration donor
|
||||
- this packet keeps `ggml-org/whisper.cpp` bounded to the narrower live
|
||||
microphone shell donor slice only
|
||||
|
||||
The granted family remains bounded to:
|
||||
|
||||
- shell panel and action contract shaping
|
||||
- donor-backed microphone shell defaults
|
||||
- session open / submit / close shell-state composition
|
||||
- speech-health capability exposure for the bounded shell
|
||||
|
||||
This packet lands only the first narrower family in that granted set.
|
||||
|
||||
## Landed scope
|
||||
|
||||
The current code now owns a retained live microphone shell contract through:
|
||||
|
||||
- retained recognition contract types for:
|
||||
- `FHyperTwistSpeechShellPanelLayout`
|
||||
- `FHyperTwistSpeechShellActionBinding`
|
||||
- `FHyperTwistSpeechMicrophoneShellProfile`
|
||||
- `FHyperTwistSpeechMicrophoneShellState`
|
||||
- microphone-shell-aware `FHyperTwistSpeechSessionConfig`
|
||||
- companion speech session state expansion in:
|
||||
- `FHyperTwistTrainingCompanionSpeechSessionState`
|
||||
- sample session-config and service-health outputs in:
|
||||
- `UHyperTwistContractLibrary`
|
||||
- direct-donor speech client seam capability exposure in:
|
||||
- `UHyperTwistMockSpeechClient`
|
||||
- `UHyperTwistHttpSpeechClient`
|
||||
- active companion shell-state normalization in:
|
||||
- `UHyperTwistTrainingSubsystem`
|
||||
- focused automation coverage in:
|
||||
- `HyperTwistWhisperCppPhase6RUMicrophoneShellContractTest.cpp`
|
||||
|
||||
## Why this is still intentionally bounded
|
||||
|
||||
This packet lands the next donor-backed microphone shell seam, but it does not
|
||||
widen into the neighboring retained families.
|
||||
|
||||
Still deferred:
|
||||
|
||||
- real device-permission workflow
|
||||
- native audio-device route ownership beyond bounded shell state
|
||||
- downloadable model or payload shipping
|
||||
- provider-profile and BYOK custody
|
||||
- voice-output ownership
|
||||
- broad assistant-platform scope
|
||||
|
||||
## Validation
|
||||
|
||||
Build validation:
|
||||
|
||||
- `C:\Program Files\Epic Games\UE_5.7\Engine\Build\BatchFiles\Build.bat UnrealHyperTwistEditor Win64 Development -Project='C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject' -WaitMutex -NoHotReloadFromIDE -NoUba`
|
||||
|
||||
Focused automation validation:
|
||||
|
||||
- `C:\Program Files\Epic Games\UE_5.7\Engine\Binaries\Win64\UnrealEditor-Cmd.exe C:\HyperTwist\UnrealHyperTwist\UnrealHyperTwist.uproject -unattended -nop4 -nosplash -NullRHI -log -stdout -FullStdOutLogOutput -AbsLog=C:\HyperTwist\UnrealHyperTwist\Saved\Logs\Phase6R-U-WhisperCpp-Verify.log -ReportExportPath=C:\HyperTwist\UnrealHyperTwist\Saved\AutomationReports\Phase6R-U-WhisperCpp-Verify -ExecCmds="Automation RunTests HyperTwist.Permissive.WhisperCpp.Phase6R.U; Quit" -TestExit="Automation Test Queue Empty"`
|
||||
|
||||
Regression automation validation:
|
||||
|
||||
- `HyperTwist.Permissive.WhisperCpp.Phase6R.E`
|
||||
- `HyperTwist.Permissive.FasterWhisper.Phase6R.F`
|
||||
- `HyperTwist.Permissive.Piper.Phase6R.G`
|
||||
- `HyperTwist.Permissive.Coqui.Phase6R.H`
|
||||
- `HyperTwist.CleanRoom.CubeDesk`
|
||||
|
||||
Expected covered tests:
|
||||
|
||||
- `MicrophoneShellSessionConfig`
|
||||
- `MicrophoneShellOpenState`
|
||||
- `MicrophoneShellUtteranceState`
|
||||
- existing `Phase 6R-E`, `Phase 6R-F`, `Phase 6R-G`, `Phase 6R-H`, and
|
||||
`CubeDesk` regression suites
|
||||
|
||||
## Queue effect
|
||||
|
||||
This packet consumes the current `Phase 6R-U` implementation slice.
|
||||
|
||||
`ggml-org/whisper.cpp` remains only partially incorporated:
|
||||
|
||||
- landed now:
|
||||
- speech transcript session boundary
|
||||
- VAD-aware session config and utterance envelope
|
||||
- transcript segment and transcript result
|
||||
- sidecar health and speech client boundary
|
||||
- live microphone shell profile and shell-state boundary
|
||||
- bounded shell capability exposure for:
|
||||
- microphone capture shell
|
||||
- streaming preview
|
||||
- manual transcript commit
|
||||
- still deferred:
|
||||
- real device-permission workflow
|
||||
- native capture-route readiness beyond bounded shell state
|
||||
- downloadable model or payload shipping
|
||||
- provider-profile and BYOK custody
|
||||
- broad assistant-platform scope
|
||||
|
||||
The next clean move is:
|
||||
|
||||
- a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and
|
||||
capture-route readiness shell preparation/control pass
|
||||
|
||||
Keep the future sequencing guards visible:
|
||||
|
||||
- `ggml-org/whisper.cpp`
|
||||
- keep code-license judgments separate from downloadable model or payload
|
||||
license review in any follow-on microphone packet
|
||||
- first-party HyperTwist
|
||||
- keep the top-level provider/session contract first-party rather than
|
||||
letting `whisper.cpp` or `faster-whisper` absorb that lane through a
|
||||
narrower donor win
|
||||
- `SYSTRAN/faster-whisper`
|
||||
- keep the row complementary to the landed shell and bounded to Python
|
||||
orchestration/service-lane ownership rather than shell ownership
|
||||
|
|
@ -0,0 +1,179 @@
|
|||
# HyperTwist Phase 6R-U whisper.cpp live microphone capture shell preparation packet
|
||||
|
||||
Created on `2026-05-24`
|
||||
|
||||
## Status
|
||||
|
||||
- historical same-day preparation authority
|
||||
- bounded post-`Phase 6R-T` control slice
|
||||
- the first bounded implementation slice now lands separately under:
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_U_WHISPER_CPP_LIVE_MICROPHONE_CAPTURE_SHELL_IMPLEMENTATION_PACKET_2026-05-24.md`
|
||||
|
||||
## Purpose
|
||||
|
||||
This packet freezes the next widening order after the landed `Phase 6R-T`
|
||||
`MagicTile` transform-aware macro remapping slice.
|
||||
|
||||
The open task was:
|
||||
|
||||
- define the next bounded source-backed widening packet for
|
||||
`ggml-org/whisper.cpp` as the retained `Phase 6R-U` live microphone capture
|
||||
shell seam
|
||||
|
||||
It is not:
|
||||
|
||||
- a full `whisper.cpp` row transplant
|
||||
- a real device-permission workflow packet
|
||||
- a downloadable model or payload-shipping packet
|
||||
- a `SYSTRAN/faster-whisper` provider/service rewrite packet
|
||||
- a `rhasspy/piper` or `coqui-ai/TTS` voice-output packet
|
||||
- a broad assistant-platform packet
|
||||
|
||||
## Current authority basis
|
||||
|
||||
This preparation packet stands on already-closed authority:
|
||||
|
||||
- `docs/REPO_LICENSE_TRACKING.md`
|
||||
- `docs/v6_5_deep_manual_pack/HyperTwist/ROADMAP.md`
|
||||
- `docs/arch/HYPERTWIST_PHASE6R_T_MAGICTILE_TRANSFORM_AWARE_MACRO_REMAPPING_IMPLEMENTATION_PACKET_2026-05-24.md`
|
||||
|
||||
The key accepted routing facts are:
|
||||
|
||||
- `ggml-org/whisper.cpp` remains the retained offline STT donor after the
|
||||
already landed `Phase 6R-E` transcript-session slice
|
||||
- `SYSTRAN/faster-whisper` remains the complementary landed Python
|
||||
orchestration donor above the existing first-party speech session boundary
|
||||
- first-party HyperTwist already owns the top-level provider/session contract,
|
||||
companion listening lifecycle, and speech-session substrate
|
||||
- the retained donor value strongest for the next narrow widening is:
|
||||
- VAD-gated streaming-shell defaults
|
||||
- rolling capture-window and keep-window posture
|
||||
- transcript-preview and manual-commit shell cues
|
||||
- health-surface capability exposure for the bounded shell
|
||||
- ownership denied here is:
|
||||
- do not widen into real device-permission workflow ownership
|
||||
- do not widen into downloadable model or payload shipping
|
||||
- do not let `whisper.cpp` inherit the top-level provider/session lane
|
||||
- do not widen into voice-output or broad assistant-platform ownership
|
||||
|
||||
## Required result
|
||||
|
||||
The source-backed control pass for this packet is now complete.
|
||||
|
||||
The first actual `6R-U` implementation packet should:
|
||||
|
||||
- define one bounded live microphone capture shell slice above the landed
|
||||
transcript-session boundary
|
||||
- land a first-party microphone shell profile and session-state surface
|
||||
- preserve donor-backed stream defaults as shell metadata rather than a broader
|
||||
runtime takeover
|
||||
- add bounded health-surface capability exposure and focused automation
|
||||
- explicitly state which neighboring retained families stay closed
|
||||
|
||||
## Source-backed retained basis
|
||||
|
||||
The queue-head decision is now source-backed rather than README-only.
|
||||
|
||||
Inspected retained donor basis:
|
||||
|
||||
- `C:\visual_studio_solutions\multi_project\GPT 5.4 HyperTwist parse\49-ggml-org-whisper-cpp-upstream-dossier.md`
|
||||
- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\stream\stream.cpp`
|
||||
- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\examples\server\README.md`
|
||||
- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\include\whisper.h`
|
||||
- `C:\Workspaces\HyperTwist\mirrors\permissive\ggml-org\whisper.cpp\tests\test-vad.cpp`
|
||||
|
||||
Inspected first-party receiving basis:
|
||||
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistSpeechClient.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingTypes.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistTraining/HyperTwistTrainingSubsystem.h`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistTraining/HyperTwistTrainingSubsystem.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockSpeechClient.cpp`
|
||||
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpSpeechClient.cpp`
|
||||
|
||||
## Narrowed 6R-U slice decision
|
||||
|
||||
The next widening slice is fixed as:
|
||||
|
||||
1. live microphone capture shell profile and session-state boundary
|
||||
|
||||
That narrowed slice covers:
|
||||
|
||||
- first-party speech shell contract types for:
|
||||
- shell panel layout
|
||||
- shell action binding
|
||||
- microphone shell profile
|
||||
- microphone shell state
|
||||
- donor-backed shell defaults for:
|
||||
- `3000ms` step window
|
||||
- `10000ms` capture window
|
||||
- `200ms` keep window
|
||||
- `0.6` VAD threshold
|
||||
- `100Hz` high-pass threshold
|
||||
- no previous-text carryover by default
|
||||
- first-party session-state composition for:
|
||||
- open-state shell activation
|
||||
- utterance preview/commit progression
|
||||
- transcript-preview, speech-gap, and last-utterance readout
|
||||
- bounded mock and HTTP speech-health capability exposure for:
|
||||
- microphone capture shell
|
||||
- streaming preview
|
||||
- manual transcript commit
|
||||
- focused automation coverage
|
||||
|
||||
## Deferred neighboring families
|
||||
|
||||
The first `6R-U` implementation packet must keep these capability families
|
||||
closed:
|
||||
|
||||
- real device-permission workflow ownership
|
||||
- native audio-device route ownership beyond bounded shell state
|
||||
- downloadable model or payload shipping
|
||||
- provider-profile or BYOK custody
|
||||
- voice-output ownership
|
||||
- broad assistant-platform scope
|
||||
|
||||
Why this slice is next:
|
||||
|
||||
- `whisper.cpp` exposes a narrower donor seam at VAD-gated streaming shell
|
||||
posture than at broader model/runtime ownership
|
||||
- the already landed `Phase 6R-E` and `Phase 6R-F` packets provide the right
|
||||
local substrate for a bounded shell layer above transcript sessions and
|
||||
orchestration metadata
|
||||
- widening directly into permission workflow, model checkout, or payload
|
||||
shipping would blur custody and legal boundaries more than clarify them at
|
||||
this point
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- the packet records why the queue advanced from landed `Phase 6R-T` to
|
||||
`Phase 6R-U`
|
||||
- the packet records that the first `6R-U` widening slice is live microphone
|
||||
capture shell profile/state only
|
||||
- the packet states exact out-of-scope families for the first `6R-U` pass
|
||||
- the packet preserves the upstream `MIT` attribution boundary explicitly
|
||||
- the packet leaves permission workflow, payload shipping, and broad
|
||||
assistant-platform ownership deferred
|
||||
|
||||
## Validation checklist
|
||||
|
||||
1. confirm the landed `Phase 6R-T` packet is the consumed prior queue head
|
||||
2. confirm the retained `whisper.cpp` seam narrows cleanly to shell profile and
|
||||
shell-state composition
|
||||
3. confirm the next implementation packet is framed as bounded source-backed
|
||||
widening rather than a broad speech-platform transplant
|
||||
|
||||
That is the packet.
|
||||
|
||||
## Queue effect
|
||||
|
||||
This preparation packet is now consumed by the landed bounded `Phase 6R-U`
|
||||
implementation slice.
|
||||
|
||||
Once that slice lands, the next clean move should stay narrow inside the
|
||||
retained `whisper.cpp` row at:
|
||||
|
||||
- a source-backed `Phase 6R-V` `ggml-org/whisper.cpp` device-permission and
|
||||
capture-route readiness shell preparation/control pass
|
||||
|
|
@ -116,14 +116,18 @@ The next bounded move is now:
|
|||
control pass is now consumed
|
||||
14. the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping
|
||||
packet is now landed in current code
|
||||
15. the next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp`
|
||||
live microphone capture shell preparation/control pass
|
||||
16. keep the speech-lane guard visible:
|
||||
15. the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell
|
||||
control pass is now consumed
|
||||
16. the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell
|
||||
packet is now landed in current code
|
||||
17. the next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp`
|
||||
device-permission and capture-route readiness shell preparation/control pass
|
||||
18. keep the speech-lane guard visible:
|
||||
- keep code-license judgments separate from model, voice, and payload-license review
|
||||
17. keep the provider-neutral speech-lane guard visible:
|
||||
19. keep the provider-neutral speech-lane guard visible:
|
||||
- first-party HyperTwist owns the top-level provider/session contract
|
||||
- `whisper.cpp` and `faster-whisper` may win narrower donor slices without inheriting that lane
|
||||
18. keep the `MagicTile` guard visible:
|
||||
20. keep the `MagicTile` guard visible:
|
||||
- keep broad non-Euclidean interaction or WinForms/OpenTK host-shell ownership closed by
|
||||
default unless a narrower first-party gap is proven above the landed macro-remapping seam
|
||||
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ Do not collapse those three tiers into one undifferentiated "features" voice.
|
|||
| Transform-aware tiling macro remapping and persistence boundary | Implemented now | landed `MagicTile` `Phase 6R-T` | First-party transform-aware remapping, reverse/setup-move playback tags, and macro XML compatibility boundary are live above the landed topology slice. |
|
||||
| Provider-backed recognition session boundary | Implemented now | first-party current code | `UHyperTwistHttpVisionClient` and normalized recognition envelopes are live first-party seams. |
|
||||
| Speech transcript session boundary | Implemented now | landed `whisper.cpp` `Phase 6R-E` | Transcript sessions, utterance envelopes, and speech-health seams are live. |
|
||||
| Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` | First-party microphone shell profile, bounded panel/action contract, and live shell-state composition are now live above the transcript-session seam. |
|
||||
| Analytics/reporting surfaces | Implemented now | landed analytics/reporting packets | Reporting is real, but bounded to accepted retained slices. |
|
||||
| Browser/spatial/media adjunct surfaces | Implemented now | landed `three.js`, `react-three-fiber`, `xr`, `model-viewer`, `remotion` packets | These are implemented bounded families, not proof of unlimited browser-shell parity. |
|
||||
|
||||
|
|
@ -197,12 +198,13 @@ repo.
|
|||
| Feature | Status | Primary authority | Notes |
|
||||
|---|---|---|---|
|
||||
| Speech transcript session boundary | Implemented now | landed `whisper.cpp` packet | Current bounded STT truth. |
|
||||
| Live microphone capture shell | Implemented now | landed `whisper.cpp` `Phase 6R-U` packet | Current bounded microphone shell truth above the landed transcript-session and orchestration seams. |
|
||||
| Provider-backed speech session health and transcript envelopes | Implemented now | first-party speech client surfaces | Current first-party speech seam. |
|
||||
| Python transcription-service orchestration profile | Implemented now | landed `faster-whisper` packet | Current bounded Python STT orchestration truth above the existing speech session boundary. |
|
||||
| Local narration sidecar seam | Implemented now | landed `piper` packet | Current bounded offline narration contract, voice catalog, and local sidecar truth. |
|
||||
| Advanced narration orchestration profile | Implemented now | landed `coqui-ai/TTS` packet | Current bounded richer narration profile above the existing companion narration contract. |
|
||||
| Broad voice output / narration sidecars | Deep-source grounded retained | `piper`, `coqui-ai/TTS` retained remainder | Keep downloadable voice/model review separate from the code-license judgment. |
|
||||
| Broad live microphone shell and downloadable payload shipping | Deep-source grounded retained | speech stack retained remainder | Explicitly deferred. |
|
||||
| Device-permission shell and downloadable payload shipping | Deep-source grounded retained | `whisper.cpp` retained remainder | Keep permission workflow, native capture-route readiness, and downloadable model/payload review separate from the landed bounded microphone shell. |
|
||||
|
||||
### 8. Provider-neutral AI/provider routing
|
||||
|
||||
|
|
|
|||
|
|
@ -87,15 +87,15 @@ Canonical discovery surfaces for roadmap interpretation:
|
|||
consumed
|
||||
- the bounded permissive `Phase 6R-E` `ggml-org/whisper.cpp` speech transcript session packet is
|
||||
now landed in current code
|
||||
- `ggml-org/whisper.cpp` remains partially incorporated; live microphone capture, model/payload
|
||||
shipping, Python orchestration, and voice-output ownership stay deferred
|
||||
- `ggml-org/whisper.cpp` remains partially incorporated; live microphone shell is now landed, but
|
||||
device-permission workflow, capture-route readiness, and model/payload shipping stay deferred
|
||||
- the later provider-family backfill keeps the top-level provider/session contract first-party and
|
||||
provider-neutral rather than donor-owned
|
||||
- the generic source-backed `Phase 6R-F` `SYSTRAN/faster-whisper` control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-F` `SYSTRAN/faster-whisper` transcription-service
|
||||
orchestration packet is now landed in current code
|
||||
- `SYSTRAN/faster-whisper` remains partially incorporated; microphone capture, BYOK/profile
|
||||
custody, payload shipping, and voice-output ownership stay deferred
|
||||
- `SYSTRAN/faster-whisper` remains partially incorporated; richer live service-lane tuning,
|
||||
BYOK/profile custody, payload shipping, and voice-output ownership stay deferred
|
||||
- the generic source-backed `Phase 6R-G` `rhasspy/piper` control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-G` `rhasspy/piper` local narration sidecar packet is now landed
|
||||
in current code
|
||||
|
|
@ -143,9 +143,13 @@ Canonical discovery surfaces for roadmap interpretation:
|
|||
control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-T` `roice3/MagicTile` transform-aware macro remapping packet
|
||||
is now landed in current code
|
||||
- the current next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp`
|
||||
live microphone capture shell preparation/control pass, while keeping speech model/payload
|
||||
review separate from the code-license judgment
|
||||
- the generic source-backed `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell
|
||||
control pass is now consumed
|
||||
- the bounded permissive `Phase 6R-U` `ggml-org/whisper.cpp` live microphone capture shell packet
|
||||
is now landed in current code
|
||||
- the current next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp`
|
||||
device-permission and capture-route readiness shell preparation/control pass, while keeping
|
||||
downloadable model/payload review separate from the code-license judgment
|
||||
- the repo-row implementation queue is now live from that `Phase 6R-A` entry point rather than
|
||||
waiting on another first-party packet
|
||||
|
||||
|
|
@ -207,14 +211,14 @@ Current routing truth:
|
|||
- active non-live implementation-board rows: `34`
|
||||
- retained benchmark, oracle, or clean-room-later rows outside the active implementation board: `9`
|
||||
|
||||
The next bounded move is a source-backed `Phase 6R-U` `ggml-org/whisper.cpp`
|
||||
live microphone capture shell preparation/control pass.
|
||||
The next bounded move is a source-backed `Phase 6R-V` `ggml-org/whisper.cpp`
|
||||
device-permission and capture-route readiness shell preparation/control pass.
|
||||
|
||||
Queue interpretation after that control pass:
|
||||
|
||||
- then continue with the retained repo-row queue
|
||||
- highest current repo-row queue pressure sits in:
|
||||
- the retained `whisper.cpp` live microphone capture shell remainder
|
||||
- the retained `whisper.cpp` device-permission and capture-route readiness shell remainder
|
||||
- `HactarCE/Hyperspeedcube` is now closed for the currently justified retained row:
|
||||
- landed:
|
||||
- puzzle catalog contract
|
||||
|
|
@ -298,11 +302,15 @@ Queue interpretation after that control pass:
|
|||
- VAD-aware session config and utterance envelope
|
||||
- transcript segment and transcript result
|
||||
- sidecar health and speech client boundary
|
||||
- live microphone shell profile and shell-state boundary
|
||||
- bounded shell capability exposure for:
|
||||
- microphone capture shell
|
||||
- streaming preview
|
||||
- manual transcript commit
|
||||
- still deferred:
|
||||
- live microphone capture shell
|
||||
- real device-permission workflow
|
||||
- native capture-route readiness beyond bounded shell state
|
||||
- downloadable model or payload shipping
|
||||
- Python orchestration / batching / hotword service layer
|
||||
- TTS or voice-output ownership
|
||||
- broad assistant-platform scope
|
||||
- `SYSTRAN/faster-whisper` is now landed as a partially incorporated row rather than a generic
|
||||
future placeholder:
|
||||
|
|
@ -311,7 +319,8 @@ Queue interpretation after that control pass:
|
|||
- orchestration-aware transcript metadata
|
||||
- bounded Python service-lane capability exposure
|
||||
- still deferred:
|
||||
- microphone capture shell
|
||||
- richer live service-lane batching, prompt-routing, or retrieval tuning beyond the landed
|
||||
bounded profile
|
||||
- provider-profile and BYOK custody
|
||||
- downloadable model or payload shipping
|
||||
- TTS or voice-output ownership
|
||||
|
|
@ -347,9 +356,9 @@ Queue interpretation after that control pass:
|
|||
- viseme / gesture runtime integration
|
||||
- broad assistant-platform scope
|
||||
- the next queue shape is now:
|
||||
- retained `whisper.cpp` live microphone capture shell assessment
|
||||
- retained `whisper.cpp` device-permission and capture-route readiness shell assessment
|
||||
- the next bounded move should stay narrow:
|
||||
- a source-backed `Phase 6R-U` control pass before any widening into downloadable
|
||||
- a source-backed `Phase 6R-V` control pass before any widening into downloadable
|
||||
model/payload shipping, broad voice-output ownership, or broad assistant-platform scope
|
||||
- keep the speech-input / voice sidecar legal sequencing guard visible:
|
||||
- keep code-license judgments separate from model, voice, and payload-license review
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue