Add provider-backed recognition sidecar client

This commit is contained in:
axiomlogicnexus 2026-05-06 03:46:41 +02:00
parent d4f4ad3401
commit 1860f1488e
8 changed files with 596 additions and 2 deletions

View file

@ -4,3 +4,13 @@ CommonButtonAcceptKeyHandling=TriggerClick
[/Script/EngineSettings.GeneralProjectSettings]
ProjectID=11C9116142A58F76A4E7F987AE989538
[/Script/UnrealHyperTwist.HyperTwistHttpVisionClient]
ProviderLabel=local-http-sidecar
ServiceBaseUrl=http://127.0.0.1:8766
OpenSessionPath=/vision/session/open
SubmitFramePath=/vision/frame
CommitObservationPath=/vision/commit
FinalizeSessionPath=/vision/session/finalize
HealthPath=/vision/health
RequestTimeoutSeconds=5.0

View file

@ -368,7 +368,9 @@ FHyperTwistVisionFinalizeResult UHyperTwistContractLibrary::MakeMockVisionFinali
FHyperTwistVisionServiceHealth UHyperTwistContractLibrary::MakeMockVisionServiceHealth()
{
FHyperTwistVisionServiceHealth Health;
Health.ProviderLabel = TEXT("mock");
Health.ServiceVersion = TEXT("mock-vision/v1");
Health.ServiceEndpoint = TEXT("in-process://mock");
Health.Capabilities = {TEXT("liveCamera3x3"), TEXT("faceCommitFlow"), TEXT("calibrationProfiles")};
Health.bReady = true;
return Health;

View file

@ -0,0 +1,402 @@
#include "HyperTwistRecognition/HyperTwistVisionClient.h"
#include "Dom/JsonObject.h"
#include "Http.h"
#include "HttpManager.h"
#include "HttpModule.h"
#include "Interfaces/IHttpRequest.h"
#include "Interfaces/IHttpResponse.h"
#include "JsonObjectConverter.h"
#include "Serialization/JsonSerializer.h"
#include "Serialization/JsonWriter.h"
namespace HyperTwistHttpVisionClientInternal
{
template <typename TStruct>
bool SerializeStruct(const TStruct& Value, FString& OutJson)
{
return FJsonObjectConverter::UStructToJsonObjectString(TStruct::StaticStruct(), &Value, OutJson, 0, 0);
}
template <typename TStruct>
bool DeserializeStruct(const FString& Json, TStruct& OutValue)
{
return !Json.IsEmpty() && FJsonObjectConverter::JsonObjectStringToUStruct(Json, &OutValue, 0, 0);
}
FString MakeSessionIdJson(const FString& SessionId)
{
FString Json;
const TSharedRef<FJsonObject> Root = MakeShared<FJsonObject>();
Root->SetStringField(TEXT("sessionId"), SessionId);
const TSharedRef<TJsonWriter<>> Writer = TJsonWriterFactory<>::Create(&Json);
FJsonSerializer::Serialize(Root, Writer);
return Json;
}
FString BuildUrl(const FString& BaseUrl, const FString& Path)
{
if (BaseUrl.IsEmpty())
{
return FString();
}
const FString TrimmedBase = BaseUrl.EndsWith(TEXT("/")) ? BaseUrl.LeftChop(1) : BaseUrl;
if (Path.IsEmpty())
{
return TrimmedBase;
}
const FString TrimmedPath = Path.StartsWith(TEXT("/")) ? Path.RightChop(1) : Path;
return FString::Printf(TEXT("%s/%s"), *TrimmedBase, *TrimmedPath);
}
struct FHttpJsonResponse
{
bool bCompleted = false;
bool bSucceeded = false;
int32 StatusCode = 0;
FString ResponseBody;
FString Error;
};
bool ExecuteJsonRequest(
const UHyperTwistHttpVisionClient& Client,
const FString& Verb,
const FString& Url,
const FString& RequestJson,
FHttpJsonResponse& OutResponse)
{
OutResponse = FHttpJsonResponse();
if (Url.IsEmpty())
{
OutResponse.Error = TEXT("service-base-url-missing");
return false;
}
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe> Request = FHttpModule::Get().CreateRequest();
Request->SetURL(Url);
Request->SetVerb(Verb);
Request->SetHeader(TEXT("Accept"), TEXT("application/json"));
Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
if (!Client.AuthorizationToken.IsEmpty())
{
Request->SetHeader(TEXT("Authorization"), FString::Printf(TEXT("Bearer %s"), *Client.AuthorizationToken));
}
if (!RequestJson.IsEmpty())
{
Request->SetContentAsString(RequestJson);
}
Request->OnProcessRequestComplete().BindLambda(
[&OutResponse](FHttpRequestPtr, FHttpResponsePtr Response, bool bWasSuccessful)
{
OutResponse.bCompleted = true;
if (Response.IsValid())
{
OutResponse.StatusCode = Response->GetResponseCode();
OutResponse.ResponseBody = Response->GetContentAsString();
}
if (!bWasSuccessful)
{
OutResponse.Error = TEXT("request-failed");
return;
}
if (!Response.IsValid())
{
OutResponse.Error = TEXT("response-missing");
return;
}
if (!EHttpResponseCodes::IsOk(OutResponse.StatusCode))
{
OutResponse.Error = FString::Printf(TEXT("http-%d"), OutResponse.StatusCode);
return;
}
OutResponse.bSucceeded = true;
});
if (!Request->ProcessRequest())
{
OutResponse.Error = TEXT("request-start-failed");
return false;
}
const double TimeoutSeconds = FMath::Max(static_cast<double>(Client.RequestTimeoutSeconds), 0.1);
const double Deadline = FPlatformTime::Seconds() + TimeoutSeconds;
while (!OutResponse.bCompleted && FPlatformTime::Seconds() < Deadline)
{
FHttpModule::Get().GetHttpManager().Tick(0.01f);
FPlatformProcess::Sleep(0.01f);
}
if (!OutResponse.bCompleted)
{
Request->CancelRequest();
OutResponse.Error = TEXT("request-timeout");
return false;
}
return OutResponse.bSucceeded;
}
}
bool UHyperTwistHttpVisionClient::OpenVisionSession(const FHyperTwistVisionSessionConfig& Config, FString& OutError)
{
OutError.Reset();
FString RequestJson;
if (!HyperTwistHttpVisionClientInternal::SerializeStruct(Config, RequestJson))
{
OutError = TEXT("request-serialize-failed");
LastTransportError = OutError;
return false;
}
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, OpenSessionPath),
RequestJson,
Response))
{
OutError = Response.Error;
LastTransportError = OutError;
return false;
}
LastTransportError.Reset();
return true;
}
FHyperTwistVisionPreviewResult UHyperTwistHttpVisionClient::SubmitVisionFrame(const FHyperTwistVisionFrameEnvelope& Frame)
{
FHyperTwistVisionPreviewResult Result;
Result.SessionId = Frame.SessionId;
Result.FrameId = Frame.FrameId;
Result.PreviewState.State.SourceSessionId = Frame.SessionId;
FString RequestJson;
if (!HyperTwistHttpVisionClientInternal::SerializeStruct(Frame, RequestJson))
{
LastTransportError = TEXT("request-serialize-failed");
Result.Guidance = TEXT("service-unavailable");
return Result;
}
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, SubmitFramePath),
RequestJson,
Response))
{
LastTransportError = Response.Error;
Result.Guidance = TEXT("service-unavailable");
return Result;
}
if (!HyperTwistHttpVisionClientInternal::DeserializeStruct(Response.ResponseBody, Result))
{
LastTransportError = TEXT("invalid-provider-response");
Result = FHyperTwistVisionPreviewResult();
Result.SessionId = Frame.SessionId;
Result.FrameId = Frame.FrameId;
Result.PreviewState.State.SourceSessionId = Frame.SessionId;
Result.Guidance = TEXT("service-unavailable");
return Result;
}
if (Result.SessionId.IsEmpty())
{
Result.SessionId = Frame.SessionId;
}
if (Result.FrameId.IsEmpty())
{
Result.FrameId = Frame.FrameId;
}
if (Result.PreviewState.State.SourceSessionId.IsEmpty())
{
Result.PreviewState.State.SourceSessionId = Frame.SessionId;
}
LastTransportError.Reset();
return Result;
}
FHyperTwistVisionCommitResult UHyperTwistHttpVisionClient::CommitVisionObservation(const FHyperTwistVisionCommitRequest& Request)
{
FHyperTwistVisionCommitResult Result;
Result.SessionId = Request.SessionId;
Result.CommitKind = Request.CommitKind;
Result.Snapshot.State.SourceSessionId = Request.SessionId;
FString RequestJson;
if (!HyperTwistHttpVisionClientInternal::SerializeStruct(Request, RequestJson))
{
LastTransportError = TEXT("request-serialize-failed");
Result.Warnings.Add(LastTransportError);
return Result;
}
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, CommitObservationPath),
RequestJson,
Response))
{
LastTransportError = Response.Error;
Result.Warnings.Add(LastTransportError);
return Result;
}
if (!HyperTwistHttpVisionClientInternal::DeserializeStruct(Response.ResponseBody, Result))
{
LastTransportError = TEXT("invalid-provider-response");
Result = FHyperTwistVisionCommitResult();
Result.SessionId = Request.SessionId;
Result.CommitKind = Request.CommitKind;
Result.Snapshot.State.SourceSessionId = Request.SessionId;
Result.Warnings.Add(LastTransportError);
return Result;
}
if (Result.SessionId.IsEmpty())
{
Result.SessionId = Request.SessionId;
}
if (Result.CommitKind.IsEmpty())
{
Result.CommitKind = Request.CommitKind;
}
if (Result.Snapshot.State.SourceSessionId.IsEmpty())
{
Result.Snapshot.State.SourceSessionId = Request.SessionId;
}
LastTransportError.Reset();
return Result;
}
FHyperTwistVisionFinalizeResult UHyperTwistHttpVisionClient::FinalizeVisionSession(const FString& SessionId)
{
FHyperTwistVisionFinalizeResult Result;
Result.SessionId = SessionId;
Result.FinalState.SourceSessionId = SessionId;
Result.FinalSnapshot.State.SourceSessionId = SessionId;
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("POST"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, FinalizeSessionPath),
HyperTwistHttpVisionClientInternal::MakeSessionIdJson(SessionId),
Response))
{
LastTransportError = Response.Error;
Result.NormalizationWarnings.Add(LastTransportError);
return Result;
}
if (!HyperTwistHttpVisionClientInternal::DeserializeStruct(Response.ResponseBody, Result))
{
LastTransportError = TEXT("invalid-provider-response");
Result = FHyperTwistVisionFinalizeResult();
Result.SessionId = SessionId;
Result.FinalState.SourceSessionId = SessionId;
Result.FinalSnapshot.State.SourceSessionId = SessionId;
Result.NormalizationWarnings.Add(LastTransportError);
return Result;
}
if (Result.SessionId.IsEmpty())
{
Result.SessionId = SessionId;
}
if (Result.FinalState.SourceSessionId.IsEmpty())
{
Result.FinalState.SourceSessionId = SessionId;
}
if (Result.FinalSnapshot.State.SourceSessionId.IsEmpty())
{
Result.FinalSnapshot.State.SourceSessionId = SessionId;
}
LastTransportError.Reset();
return Result;
}
FHyperTwistVisionServiceHealth UHyperTwistHttpVisionClient::GetVisionServiceHealth() const
{
FHyperTwistVisionServiceHealth Health;
Health.ProviderLabel = ProviderLabel;
Health.ServiceVersion = TEXT("provider-unavailable/v1");
Health.ServiceEndpoint = ServiceBaseUrl;
Health.bReady = false;
Health.Capabilities = {
TEXT("transport:http"),
TEXT("provider-backed-recognition")
};
HyperTwistHttpVisionClientInternal::FHttpJsonResponse Response;
if (!HyperTwistHttpVisionClientInternal::ExecuteJsonRequest(
*this,
TEXT("GET"),
HyperTwistHttpVisionClientInternal::BuildUrl(ServiceBaseUrl, HealthPath),
FString(),
Response))
{
LastTransportError = Response.Error;
Health.LastError = LastTransportError;
return Health;
}
if (!HyperTwistHttpVisionClientInternal::DeserializeStruct(Response.ResponseBody, Health))
{
LastTransportError = TEXT("invalid-provider-response");
Health = FHyperTwistVisionServiceHealth();
Health.ProviderLabel = ProviderLabel;
Health.ServiceVersion = TEXT("provider-unavailable/v1");
Health.ServiceEndpoint = ServiceBaseUrl;
Health.bReady = false;
Health.Capabilities = {
TEXT("transport:http"),
TEXT("provider-backed-recognition")
};
Health.LastError = LastTransportError;
return Health;
}
if (Health.ProviderLabel.IsEmpty())
{
Health.ProviderLabel = ProviderLabel;
}
if (Health.ServiceEndpoint.IsEmpty())
{
Health.ServiceEndpoint = ServiceBaseUrl;
}
Health.Capabilities.AddUnique(TEXT("transport:http"));
Health.Capabilities.AddUnique(TEXT("provider-backed-recognition"));
Health.LastError.Reset();
LastTransportError.Reset();
return Health;
}

View file

@ -71,6 +71,8 @@ FHyperTwistVisionFinalizeResult UHyperTwistMockVisionClient::FinalizeVisionSessi
FHyperTwistVisionServiceHealth UHyperTwistMockVisionClient::GetVisionServiceHealth() const
{
FHyperTwistVisionServiceHealth Health = UHyperTwistContractLibrary::MakeMockVisionServiceHealth();
Health.ProviderLabel = TEXT("mock");
Health.ServiceEndpoint = TEXT("in-process://mock");
Health.Capabilities.Add(TEXT("sessioned-mock-client"));
return Health;
}

View file

@ -169,12 +169,21 @@ struct FHyperTwistVisionServiceHealth
{
GENERATED_BODY()
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ProviderLabel = TEXT("mock");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ServiceVersion = TEXT("mock-vision/v1");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString ServiceEndpoint;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
TArray<FString> Capabilities;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
FString LastError;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "HyperTwist")
bool bReady = true;
};

View file

@ -42,7 +42,7 @@ public:
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
UFUNCTION(BlueprintPure, Category = "HyperTwist|Recognition")
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;
private:
@ -52,3 +52,55 @@ private:
UPROPERTY()
TMap<FString, int32> SessionFrameCounts;
};
UCLASS(BlueprintType, Config = Game, DefaultConfig)
class UNREALHYPERTWIST_API UHyperTwistHttpVisionClient : public UObject, public IHyperTwistVisionClient
{
GENERATED_BODY()
public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString ProviderLabel = TEXT("local-http-sidecar");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString ServiceBaseUrl = TEXT("http://127.0.0.1:8766");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString OpenSessionPath = TEXT("/vision/session/open");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString SubmitFramePath = TEXT("/vision/frame");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString CommitObservationPath = TEXT("/vision/commit");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString FinalizeSessionPath = TEXT("/vision/session/finalize");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString HealthPath = TEXT("/vision/health");
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
float RequestTimeoutSeconds = 5.0f;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Config, Category = "HyperTwist|Recognition")
FString AuthorizationToken;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual bool OpenVisionSession(const FHyperTwistVisionSessionConfig& Config, FString& OutError) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionPreviewResult SubmitVisionFrame(const FHyperTwistVisionFrameEnvelope& Frame) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionCommitResult CommitVisionObservation(const FHyperTwistVisionCommitRequest& Request) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionFinalizeResult FinalizeVisionSession(const FString& SessionId) override;
UFUNCTION(BlueprintCallable, Category = "HyperTwist|Recognition")
virtual FHyperTwistVisionServiceHealth GetVisionServiceHealth() const override;
private:
mutable FString LastTransportError;
};

View file

@ -10,7 +10,7 @@ public class UnrealHyperTwist : ModuleRules
PublicDependencyModuleNames.AddRange(new string[] { "Core", "CoreUObject", "Engine", "InputCore", "EnhancedInput", "Json", "JsonUtilities", "UMG" });
PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" });
PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore", "HTTP" });
// Uncomment if you are using Slate UI
// PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" });

View file

@ -0,0 +1,117 @@
# HyperTwist Phase 4 provider-backed recognition packet
Created on `2026-05-06`
Status:
- first-party HyperTwist packet
- bounded Phase `4` recognition productionization slice
## Purpose
This packet closes the first honest recognition-productionization move after the orchestration entry lane landed.
The open task is:
- add one provider-backed recognition client behind the existing `IHyperTwistVisionClient` contract
It is not:
- a gameplay-facing recognition UX packet
- a speech or simulation packet
- a provider lock-in decision
- a websocket/live-stream packet
## Scope
Bounded lane:
- keep the existing mock client intact
- add a local-first HTTP sidecar implementation over the same vision contract
- make the provider-backed client config-driven from the Unreal project config
- surface enough health metadata to inspect which provider lane is active and whether transport failed
Out of scope:
- changing coach/training consumers to depend on recognition now
- broad recognition UI
- websocket grammar or continuous frame streaming
- exact state-validation deepening inside `HyperTwistCore`
## Why this is the right next packet
Current code reality before this packet:
- `HyperTwistRecognition` already had an owned sidecar contract
- replay-facing recognition summary derivation already existed
- the only runtime implementation was the in-process mock client
That meant recognition was contract-strong but still not provider-backed.
The bounded fix is:
- add one real sidecar transport implementation without changing the owned contract shape
## What landed
Primary code changes:
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistRecognitionTypes.h`
- expanded `FHyperTwistVisionServiceHealth` with:
- `ProviderLabel`
- `ServiceEndpoint`
- `LastError`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Public/HyperTwistRecognition/HyperTwistVisionClient.h`
- added `UHyperTwistHttpVisionClient`
- class is `Config=Game` / `DefaultConfig`
- config surface includes:
- `ServiceBaseUrl`
- operation paths
- timeout
- optional bearer token
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistHttpVisionClient.cpp`
- added the provider-backed implementation
- uses Unreal HTTP plus JSON struct serialization
- posts the existing owned request payloads and reads the existing owned response payloads
- keeps transport synchronous and bounded for the current non-streaming contract stage
- `UnrealHyperTwist/Config/DefaultGame.ini`
- added the initial local-first sidecar config section
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistRecognition/HyperTwistMockVisionClient.cpp`
- `UnrealHyperTwist/Source/UnrealHyperTwist/Private/HyperTwistBootstrap/HyperTwistContractLibrary.cpp`
- refreshed mock health metadata so mock and provider-backed health share the same inspection fields
- `UnrealHyperTwist/Source/UnrealHyperTwist/UnrealHyperTwist.Build.cs`
- added the Unreal `HTTP` module dependency
## Transport decision
The sidecar contract already said:
- websocket preferred for live frame flow
- HTTP acceptable for batch or non-streaming calls
This packet intentionally chooses:
- HTTP
because it is the smallest productionization move that:
- keeps the local-first sidecar boundary
- preserves provider neutrality
- does not widen into a live-streaming runtime rewrite
## Acceptance criteria
- `IHyperTwistVisionClient` still owns the recognition boundary
- a provider-backed implementation now exists in first-party code
- the provider-backed implementation is config-driven rather than hard-coded into a consumer
- the mock client still works
- full product build succeeds
## Validation checklist
1. build `UnrealHyperTwist.sln`
2. confirm the new HTTP client compiles and links
3. confirm the mock health shape still compiles with the expanded metadata
4. confirm no first-party training/coaching packet was reopened
That is the packet.