Compare commits

...

2 Commits

Author SHA1 Message Date
DuOtto c9e46eeb41 Added README 2026-07-27 09:32:25 +02:00
DuOtto aed58897cf Added Gesture Actions, Gesture Definitions, TableIcons, ... 2026-07-15 22:09:56 +02:00
29 changed files with 1109 additions and 214 deletions
-2
View File
@@ -4,8 +4,6 @@
"Component.Unreal.Debugger",
"Component.Unreal.Ide",
"Microsoft.Net.Component.4.6.2.TargetingPack",
"Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL",
"Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64",
"Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL",
"Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64",
"Microsoft.VisualStudio.Component.VC.Llvm.Clang",
+65
View File
@@ -0,0 +1,65 @@
<Solution>
<Configurations>
<BuildType Name="Analyze" />
<BuildType Name="Debug" />
<BuildType Name="Development" />
<BuildType Name="Release" />
</Configurations>
<Folder Name="/Dependencies/">
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Analytics.Telemetry/EpicGames.Analytics.Telemetry.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build/EpicGames.Build.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph/EpicGames.BuildGraph.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core/EpicGames.Core.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde/EpicGames.Horde.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash/EpicGames.IoHash.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers/EpicGames.LogMatchers.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.MsBuild/EpicGames.MsBuild.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.OIDC/EpicGames.OIDC.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Oodle/EpicGames.Oodle.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Fixture/EpicGames.Perforce.Fixture.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed/EpicGames.Perforce.Managed.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce/EpicGames.Perforce.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.ProjectStore/EpicGames.ProjectStore.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization/EpicGames.Serialization.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UBA/EpicGames.UBA.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UHT/EpicGames.UHT.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/UnrealBuildTool/UnrealBuildTool.csproj" />
</Folder>
<Folder Name="/Tests/">
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build.Tests/EpicGames.Build.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph.Tests/EpicGames.BuildGraph.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core.Tests/EpicGames.Core.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde.Tests/EpicGames.Horde.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash.Tests/EpicGames.IoHash.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers.Tests/EpicGames.LogMatchers.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed.Tests/EpicGames.Perforce.Managed.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Tests/EpicGames.Perforce.Tests.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization.Tests/EpicGames.Serialization.Tests.csproj" />
</Folder>
<Project Path="G:/Unreal/UE_5.8/Engine/Platforms/VisionOS/Source/Programs/AutomationTool/VisionOS.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Android/Android.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Apple/Apple.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationTool.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationUtils/AutomationUtils.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/BuildGraph/BuildGraph.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CookedEditor/CookedEditor.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CrowdinLocalization/CrowdinLocalization.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Gauntlet/Gauntlet.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Insights/Insights.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/IOS/IOS.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Linux/Linux.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LiveLinkHub/LiveLinkHub.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Localization/Localization.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LowLevelTests/LowLevelTests.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mac/Mac.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Microsoft/Microsoft.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mutable/Mutable.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/OneSkyLocalization/OneSkyLocalization.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Scripts/AutomationScripts.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SmartlingLocalization/SmartlingLocalization.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SteamDeck/SteamDeck.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Turnkey/Turnkey.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/TVOS/TVOS.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Win/Win.Automation.csproj" />
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/XLocLocalization/XLocLocalization.Automation.csproj" />
</Solution>
Binary file not shown.
+68
View File
@@ -0,0 +1,68 @@
# Table-UE
> **Work in progress.** This is an actively developed personal project, not a finished product — expect incomplete features, rough edges, and breaking changes.
Unreal Engine side of a real-time, gesture-controlled smart tabletop for tabletop wargaming (Warhammer 40k). A downward-facing multi-camera rig tracks players' hands over the physical table; this project receives the 3D hand data over UDP, recognizes gestures, and drives an interactive projected overlay (rulers, markers, HUDs, etc.) directly on the tabletop surface.
The companion computer-vision/UI application that captures and streams the hand-tracking data lives in [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python).
## Overview
- Built with **Unreal Engine 5.8** (C++ gameplay module + Blueprints).
- Receives per-frame 3D hand landmark data (21 points/hand, MediaPipe topology) via a **raw UDP socket**, decoded into `FHandPacketHand` structs.
- Spawns and updates `AHandActor` instances per tracked hand and exposes them to Blueprints for visualization/debugging.
- A **JSON-driven gesture binding system** matches live hand poses against configurable gesture definitions (e.g. "point with index finger") and triggers corresponding **Gesture Actions**.
- Gesture Actions are self-contained, extendable Actors that own their own on-table visuals ("Table Icons") — e.g. a `Ruler` gesture action that draws a measuring line with tick marks between two points and displays the distance in real-world units.
- A calibration scale factor (`UEUnitsPerUnit`) converts between real-world measurements (inches/cm on the physical table) and Unreal units, set up per session.
## Architecture
```
Table-Python (multi-camera capture, MediaPipe hand tracking, stereo triangulation)
│ UDP (custom binary hand packet protocol)
ADetectionManager — UDP receiver, decodes packets into hand data
AHandManager — spawns/despawns AHandActor per tracked hand id
UGestureManagerComponent — loads gesture definitions from JSON, evaluates hand poses each tick
AGestureAction (+ subclasses, e.g. AGestureAction_Ruler)
ATableIconActor (+ subclasses) — the actual on-table visuals (lines, labels, procedural meshes, UMG widgets)
```
### Key classes
| Class | Responsibility |
|---|---|
| [`ADetectionManager`](Source/Table/Public/DetectionManager.h) | Owns the UDP socket, parses incoming hand-tracking packets |
| [`AHandManager`](Source/Table/Public/HandManager.h) | Tracks the set of currently-active hands, broadcasts added/removed delegates |
| [`AHandActor`](Source/Table/Public/HandActor.h) / [`FingerTip`](Source/Table/Public/FingerTip.h) | Represents a single tracked hand and its fingertip positions in world space |
| [`UGestureManagerComponent`](Source/Table/Public/GestureManagerComponent.h) | Loads a JSON gesture-binding config and evaluates gestures against tracked hands |
| [`UGestureDefinition`](Source/Table/Public/GestureDefinition.h) | Scores how well a hand pose matches a named gesture (overridden per gesture) |
| [`AGestureAction`](Source/Table/Public/GestureAction.h) / [`AGestureAction_Ruler`](Source/Table/Public/MyGestureActions.h) | Lifecycle (init/update/end) for an active gesture, e.g. a ruler measurement |
| [`ATableIconActor`](Source/Table/Public/TableIconActor.h) / [`ATableIconLineActor`](Source/Table/Public/MyTableIconActors.h) | Visual elements drawn on the table for the duration of a gesture |
Gesture-to-action bindings are configured in JSON (see `Content/WorldObjects/GestureActions/`), so new gestures can be wired up without touching C++.
## Requirements
- Unreal Engine **5.8**
- Visual Studio 2022 (Windows) with the "Game development with C++" workload
- A running instance of [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python) sending hand-tracking data over UDP (default `127.0.0.1:9000`) if you want live hand input; the level can otherwise be explored/tested without a live feed.
## Getting started
1. Clone the repo and generate project files by right-clicking `Table.uproject`**Generate Visual Studio project files** (or open it directly in the Unreal Editor, which will prompt to build missing modules).
2. Open `Table.sln` in Visual Studio and build, or open `Table.uproject` in Unreal Editor.
3. Press **Play** to start the `ADetectionManager` UDP listener.
4. Start the Python tracking application to begin streaming live hand data to the table.
## Status
Personal/hobby project, actively evolving — gesture set, table icons, and HUD content are still being expanded.
+2 -2
View File
@@ -8,8 +8,8 @@ public class TableTarget : TargetRules
public TableTarget(TargetInfo Target) : base(Target)
{
Type = TargetType.Game;
DefaultBuildSettings = BuildSettingsVersion.V6;
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
DefaultBuildSettings = BuildSettingsVersion.V7;
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
ExtraModuleNames.Add("Table");
}
}
+29 -11
View File
@@ -3,44 +3,62 @@
#include "GestureAction.h"
#include "HandActor.h"
#include "TableIconActor.h"
// Sets default values
AGestureAction::AGestureAction()
{
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = true;
PrimaryActorTick.bCanEverTick = false;
}
// Called when the game starts or when spawned
void AGestureAction::BeginPlay()
{
Super::BeginPlay();
}
// Called every frame
void AGestureAction::Tick(float DeltaTime)
void AGestureAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
{
Super::Tick(DeltaTime);
HandA = Hand;
bRequiresSecondHand = Binding.LeftGesture != "" && Binding.RightGesture != "";
bExtendable = Binding.ExtendedGestures.IsEmpty();
Binding = NewBinding;
HandManager = Hand->HandManager;
UEUnitsPerUnit = NewUEUnitsPerUnit;
}
void AGestureAction::UpdateAction()
void AGestureAction::UpdateAction_Implementation()
{
Destroy();
}
void AGestureAction::AbortAction()
void AGestureAction::AbortAction_Implementation()
{
for (ATableIconActor* Icon : Icons)
Icon->bPersistOnActionEnd = false;
EndGesture();
}
void AGestureAction::AddExtension(AHandActor* OtherHand)
void AGestureAction::AddExtension_Implementation(AGestureAction* OtherAction)
{
LinkedActions = OtherAction;
}
void AGestureAction::AddOtherHand_Implementation(AHandActor* OtherHand)
{
HandB = OtherHand;
}
void AGestureAction::EndGesture()
void AGestureAction::EndGesture_Implementation()
{
DestroyIcons();
Destroy();
}
void AGestureAction::DestroyIcons()
{
for (ATableIconActor* Icon : Icons)
if (IsValid(Icon) ) Icon->OnActionEnded();
Icons.Empty();
}
@@ -40,6 +40,11 @@ const FVector& UGestureDefinition::GetPoint(const AHandActor* Hand, int32 Index)
return Hand->Points[Index];
}
bool UGestureDefinition::IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold) const
{
return GetFingerExtendedScore(Hand, Finger) > Threshold;
}
float UGestureDefinition::GetHandScale(const AHandActor* Hand) const
{
const TArray<FVector>& P = Hand->Points;
@@ -53,30 +58,44 @@ float UGestureDefinition::GetNormalizedDistance(const AHandActor* Hand, int32 A,
}
float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const
{
float Dist = GetNormalizedDistance(
Hand,
GetFingerTipIndex(A),
GetFingerTipIndex(B)
);
const FVector& P1 = Hand->Points[GetFingerTipIndex(A)];
const FVector& P2 = Hand->Points[GetFingerTipIndex(B)];
float Dist = FVector::Dist(P1, P2);
float HandScale = FVector::Dist( Hand->Points[0], Hand->Points[9] );
float Normalized = Dist / HandScale;
//UE_LOG(LogTemp, Warning, TEXT("Dist: %.3f | HandScale: %.3f | Norm: %.3f"), Dist, HandScale, Normalized);
const float Threshold = 0.25f;
return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f);
return 1.f - FMath::Clamp( Normalized / Threshold, 0.f, 1.f );
}
float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const
{
const TArray<FVector>& P = Hand->Points;
if (Finger == EHandFinger::Thumb)
{
// Thumb: compare tip->base against index base->wrist as reference direction
const FVector& Base = P[2];
const FVector& Tip = P[4];
const FVector& Ref = P[5]; // index base as stable reference
float Dot = FVector::DotProduct(
(Tip - Base).GetSafeNormal(),
(Ref - P[0]).GetSafeNormal());
return FMath::Clamp(Dot, 0.f, 1.f);
}
const FVector& Wrist = P[0];
const FVector& Base = P[GetFingerBaseIndex(Finger)];
const FVector& Tip = P[GetFingerTipIndex(Finger)];
float Dot = FVector::DotProduct(
(Tip - Base).GetSafeNormal(),
(Base - Wrist).GetSafeNormal()
);
(Base - Wrist).GetSafeNormal());
return FMath::Clamp(Dot, 0.f, 1.f);
}
@@ -91,9 +110,7 @@ float UGestureDefinition::GetFingersAverage(const AHandActor* Hand, const TArray
for (EHandFinger Finger : Fingers)
{
Sum += bExtended
? GetFingerExtendedScore(Hand, Finger)
: GetFingerCurledScore(Hand, Finger);
Sum += bExtended ? GetFingerExtendedScore(Hand, Finger) : GetFingerCurledScore(Hand, Finger);
}
return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f;
@@ -107,19 +124,57 @@ FVector UGestureDefinition::GetFingerDirection(const AHandActor* Hand, EHandFing
return (Tip - Base).GetSafeNormal();
}
float UGesture_PinchPoint::Evaluate(AHandActor* Hand) const
float UGesture_PinchIndex::Evaluate(AHandActor* Hand) const
{
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
float IndexExtended = GetFingerExtendedScore(Hand, EHandFinger::Index);
float OthersCurled = GetFingersAverage(
Hand,
{ EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky },
false
);
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky }, false );
return
Pinch * 0.5f +
IndexExtended * 0.3f +
OthersCurled * 0.2f;
return Pinch * 0.5f + IndexExtended * 0.3f + OthersCurled * 0.2f;
}
float UGesture_PinchMiddleIndex::Evaluate(AHandActor* Hand) const
{
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Middle);
Pinch += GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
float MiddleExtended = GetFingerExtendedScore(Hand, EHandFinger::Middle);
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Ring, EHandFinger::Pinky }, false );
return Pinch * 0.5f + MiddleExtended * 0.3f + OthersCurled * 0.2f;
}
float UGesture_PointIndex::Evaluate(AHandActor* Hand) const
{
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
float Middle = GetFingerCurledScore(Hand, EHandFinger::Middle);
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
return ( Index + Middle + Ring + Pinky + Thumb ) / 5.f;
}
float UGesture_PointMiddle::Evaluate(AHandActor* Hand) const
{
float Index = GetFingerCurledScore(Hand, EHandFinger::Index);
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
}
float UGesture_PointIndexMiddle::Evaluate(AHandActor* Hand) const
{
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
}
@@ -14,7 +14,7 @@ UGestureManagerComponent::UGestureManagerComponent()
// off to improve performance if you don't need them.
PrimaryComponentTick.bCanEverTick = false;
HandManager = Cast<AHandManager>(GetOwner());
}
@@ -23,6 +23,8 @@ void UGestureManagerComponent::BeginPlay()
{
Super::BeginPlay();
HandManager = Cast<AHandManager>(GetOwner());
HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded);
HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved);
@@ -42,17 +44,16 @@ void UGestureManagerComponent::EvaluateHands()
if (Hand->bUsedThisTick) continue;
switch (Hand->Phase)
{
case EGesturePhase::Started:
case EHandGesturePhase::Started:
TriggerAction(Hand);
break;
case EGesturePhase::Ended:
case EHandGesturePhase::Ended:
Hand->ResetGesture();
EndGesture();
break;
}
}
UpdateOngoingAction();
UpdateOngoingActions();
}
void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
@@ -62,14 +63,16 @@ void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand)
{
Hand->PairedHand->PairedHand = nullptr;
Hand->ResetGesture();
if (!Hand->ActiveAction) return;
if (!ActiveGestureActions.Contains(Hand->ActiveAction)) return;
Hand->ActiveAction->Destroy();
ActiveGestureActions.Remove(Hand->ActiveAction);
}
void UGestureManagerComponent::LoadBindings(const FString& FilePath)
{
FString JsonString;
FString JsonString = "";
FString Temp = "";
if (!FFileHelper::LoadFileToString(JsonString, *FilePath))
{
return;
@@ -92,9 +95,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
FGestureBinding Binding;
Obj->TryGetStringField(TEXT("action"), Binding.ActionName);
Obj->TryGetStringField(TEXT("left"), Binding.LeftGesture);
Obj->TryGetStringField(TEXT("right"), Binding.RightGesture);
Obj->TryGetStringField(TEXT("action"), Temp);
Binding.ActionName = FName(Temp);
Obj->TryGetStringField(TEXT("left"), Temp);
Binding.LeftGesture = FName(Temp);
Obj->TryGetStringField(TEXT("right"), Temp);
Binding.RightGesture = FName(Temp);
const TArray<TSharedPtr<FJsonValue>>* ExtArray;
if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray))
@@ -106,10 +112,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
FExtendedGestureBinding Ext;
ExtObj->TryGetStringField(TEXT("gesture"), Ext.GestureName);
ExtObj->TryGetStringField(TEXT("action"), Ext.GestureAction);
ExtObj->TryGetStringField(TEXT("gesture"), Temp);
Ext.GestureName = FName(Temp);
ExtObj->TryGetStringField(TEXT("action"), Temp);
Ext.GestureAction = FName(Temp);
Binding.ExentdedGestures.Add(Ext);
Binding.ExtendedGestures.Add(Ext);
}
}
@@ -143,6 +151,7 @@ void UGestureManagerComponent::BuildDefinitionMap()
UGestureDefinition* Def = NewObject<UGestureDefinition>(this, Row->DefinitionClass);
DefinitionMap.Add(Def->GestureName, Def);
GestureDefinitions.Add(Def);
}
}
@@ -159,176 +168,171 @@ void UGestureManagerComponent::BuildActionMap()
{
if (!Row || !Row->ActionClass) continue;
ActionClassMap.Add(
Row->ActionClass->GetName(),
Row->ActionClass
);
ActionClassMap.Add( FName(Row->ActionClass->GetName()), Row->ActionClass );
}
}
void UGestureManagerComponent::CheckGestureDefinitions(TArray<AHandActor*>& Hands)
{
const float MinScoreThreshold = 0.5f;
//const float MinScoreThreshold = 0.7f;
for (AHandActor* Hand : Hands) {
float Threshold = (Hand->Phase == EHandGesturePhase::Ongoing || Hand->Phase == EHandGesturePhase::Started) ? 0.60f : 0.75f;
float BestScore = 0.f;
FString BestGesture = "";
FName BestGesture = "";
for (UGestureDefinition* Gesture : GestureDefinitions) {
float Score = Gesture->Evaluate(Hand);
float Score = Gesture->Evaluate(Hand); // Getting the Score of the gestures to check
if (Score > BestScore)
//UE_LOG(LogTemp, Warning, TEXT("%s -> %s : %.3f"), *Hand->GetName(), *Gesture->GestureName, Score);
if (Score > BestScore) // Getting the gesture with the highest Score
{
BestScore = Score;
BestGesture = Gesture->GestureName;
}
}
if (BestScore < MinScoreThreshold)
// Check if the gesture with the highest score is a valid Gesture
if (BestScore < Threshold)
{
if (Hand->Phase != EGesturePhase::None)
if (Hand->Phase != EHandGesturePhase::None)
{
Hand->FailFrames++;
UE_LOG(LogTemp, Warning,
TEXT("[%s] Below threshold (%s) Score=%.3f Fail=%d/%d"),
*Hand->GetName(),
*Hand->ActiveGestureName.ToString(),
BestScore,
Hand->FailFrames,
FailThreshold);
//if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
if (Hand->FailFrames >= FailThreshold)
{
Hand->Phase = EGesturePhase::Ended;
UE_LOG(LogTemp, Warning,
TEXT("[%s] -> ENDED (%s)"),
*Hand->GetName(),
*Hand->ActiveGestureName.ToString());
Hand->Phase = EHandGesturePhase::Ended;
}
}
continue;
}
// Handle valid Gesture
switch (Hand->Phase)
{
case EGesturePhase::None:
case EHandGesturePhase::None: // Starting a Candidate
Hand->ActiveGestureName = BestGesture;
Hand->Phase = EGesturePhase::Candidate;
Hand->Phase = EHandGesturePhase::Candidate;
Hand->CandidateFrames = 1;
Hand->FailFrames = 0;
UE_LOG(LogTemp, Warning,
TEXT("[%s] NONE -> CANDIDATE (%s)"),
*Hand->GetName(),
*BestGesture.ToString());
break;
case EGesturePhase::Candidate:
if (Hand->ActiveGestureName != BestGesture)
case EHandGesturePhase::Candidate: // Checking if the Candidate stays valid or another gesture is made
if (Hand->ActiveGestureName == BestGesture)
{
Hand->FailFrames++;
break;
Hand->CandidateFrames++;
Hand->FailFrames = 0;
UE_LOG(LogTemp, Warning,
TEXT("[%s] Candidate %s (%d/%d)"),
*Hand->GetName(),
*BestGesture.ToString(),
Hand->CandidateFrames,
CandidateThreshold);
//if (Hand->CandidateFrames >= CandidateThreshold) Hand->Phase = EHandGesturePhase::Started;
if (Hand->CandidateFrames >= CandidateThreshold)
{
Hand->Phase = EHandGesturePhase::Started;
UE_LOG(LogTemp, Warning,
TEXT("[%s] CANDIDATE -> STARTED (%s)"),
*Hand->GetName(),
*BestGesture.ToString());
}
break;
}
UE_LOG(LogTemp, Warning,
TEXT("[%s] Candidate switched %s -> %s"),
*Hand->GetName(),
*Hand->ActiveGestureName.ToString(),
*BestGesture.ToString());
Hand->CandidateFrames++;
Hand->ActiveGestureName = BestGesture;
Hand->CandidateFrames = 1;
Hand->FailFrames = 0;
if (Hand->CandidateFrames >= CandidateThreshold)
Hand->Phase = EGesturePhase::Started;
break;
case EGesturePhase::Ongoing:
case EHandGesturePhase::Ongoing: // Checking if the gesture is still ongoing
//if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
//else Hand->FailFrames = 0;
if (Hand->ActiveGestureName != BestGesture)
{
Hand->FailFrames++;
if (Hand->FailFrames >= FailThreshold)
{
Hand->Phase = EGesturePhase::Ended;
}
UE_LOG(LogTemp, Warning,
TEXT("[%s] Ongoing fail %s (%d/%d)"),
*Hand->GetName(),
*Hand->ActiveGestureName.ToString(),
Hand->FailFrames,
FailThreshold);
}
else
{
Hand->FailFrames = 0;
if (Hand->FailFrames > 0)
{
UE_LOG(LogTemp, Warning,
TEXT("[%s] Ongoing recovered %s"),
*Hand->GetName(),
*Hand->ActiveGestureName.ToString());
}
}
break;
//case EHandGesturePhase::Started:
// break;
default: // If the gesture would be stuck somewhere
if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
if (Hand->FailFrames >= FailThreshold) Hand->ResetGesture(); // This optimaly shouldnt be called here
break;
}
if (Hand->FailFrames >= FailThreshold)
Hand->Phase = EGesturePhase::Ended;
// Checking if a gesture has ended
if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
}
}
void UGestureManagerComponent::TriggerAction(AHandActor* Hand)
{
if (!Hand->bUsedThisTick) return;
if (Hand->IsPaired())
{
AHandActor* Other = Hand->PairedHand;
for (const FGestureBinding& Binding : GestureBindings)
{
const bool bLeftMatch = (Hand->Handedness == EHandedness::Left && Hand->ActiveGestureName == Binding.LeftGesture &&
Other->ActiveGestureName == Binding.RightGesture);
const bool bRightMatch = (Hand->Handedness == EHandedness::Right && Hand->ActiveGestureName == Binding.RightGesture &&
Other->ActiveGestureName == Binding.LeftGesture);
if (!(bLeftMatch || bRightMatch))
continue;
// both must be ready
if (Other->Phase != EGesturePhase::Started)
continue;
// mark used
Hand->bUsedThisTick = true;
Other->bUsedThisTick = true;
Hand->Phase = EGesturePhase::Ongoing;
Other->Phase = EGesturePhase::Ongoing;
// spawn action (centralized factory recommended)
SpawnGestureAction(Binding.ActionName, Hand, Other);
return;
}
}
if (Hand->bUsedThisTick) return;
if (TryExtendGestureAction(Hand)) return;
for (const FGestureBinding& Binding : GestureBindings)
{
const bool bMatch =
(Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName) ||
(Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
if (!bMatch)
continue;
if (!bMatches) continue;
// if paired hand is actively running something, allow extension check later
if (Hand->IsPaired() && Hand->PairedHand && Hand->PairedHand->Phase == EGesturePhase::Ongoing)
{
for (const FExtendedGestureBinding& Ext : Binding.ExentdedGestures)
{
if (Ext.GestureName != Hand->ActiveGestureName)
continue;
AGestureAction* NewAction = SpawnGestureAction(Binding.ActionName, Hand);
NewAction->InitAction(Hand, Binding, UEUnitsPerUnit);
if (Ext.GestureAction == Binding.ActionName)
{
// extend current action
ExtendGestureAction(Binding.ActionName, Hand);
Hand->bUsedThisTick = true;
return;
}
else
{
// branch: spawn new action linked to existing one
SpawnGestureActionWithParent(Ext.GestureAction, Hand, Hand->PairedHand);
Hand->bUsedThisTick = true;
return;
}
}
}
// normal one-hand action
Hand->bUsedThisTick = true;
SpawnGestureAction(Binding.ActionName, Hand, nullptr);
return;
}
}
void UGestureManagerComponent::UpdateOngoingAction()
void UGestureManagerComponent::UpdateOngoingActions()
{
for (AGestureAction* Action : ActiveGestureActions)
{
@@ -338,18 +342,94 @@ void UGestureManagerComponent::UpdateOngoingAction()
}
}
void UGestureManagerComponent::SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B)
AGestureAction* UGestureManagerComponent::SpawnGestureAction(FName ActionName, AHandActor* Hand)
{
TSubclassOf<AGestureAction>* ActionClassPtr = ActionClassMap.Find(ActionName);
if (!ActionClassPtr || !Hand) return nullptr;
FActorSpawnParameters SpawnParams;
SpawnParams.Owner = GetOwner();
SpawnParams.SpawnCollisionHandlingOverride =
ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
*ActionClassPtr,
FVector::ZeroVector,
FRotator::ZeroRotator,
SpawnParams);
if (!NewAction) return nullptr;
Hand->ActiveAction = NewAction;
Hand->bUsedThisTick = true;
Hand->Phase = EHandGesturePhase::Ongoing;
return NewAction;
}
void UGestureManagerComponent::ExtendGestureAction(FString ActionName, AHandActor* Hand)
bool UGestureManagerComponent::TryExtendGestureAction(AHandActor* Hand)
{
if (!Hand->IsPaired() || Hand->PairedHand->Phase != EHandGesturePhase::Ongoing || !Hand->PairedHand->ActiveAction) return false;
if (TryJoinExistingAction(Hand)) return true;
if (TryCreateExtension(Hand)) return true;
return false;
}
void UGestureManagerComponent::SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand)
bool UGestureManagerComponent::TryJoinExistingAction(AHandActor* Hand)
{
AHandActor* OtherHand = Hand->PairedHand;
AGestureAction* OtherAction = OtherHand->ActiveAction;
if (!OtherAction->bRequiresSecondHand) return false; // Check if its a Two Handed Action or an extension
FGestureBinding& Binding = OtherAction->Binding;
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
if (!bMatches) return false;
OtherAction->AddOtherHand(Hand);
OtherHand->bUsedThisTick = true;
Hand->bUsedThisTick = true;
Hand->Phase = EHandGesturePhase::Ongoing;
Hand->ActiveAction = OtherAction;
return true;
}
bool UGestureManagerComponent::TryCreateExtension(AHandActor* Hand)
{
AHandActor* OtherHand = Hand->PairedHand;
AGestureAction* OtherAction = OtherHand->ActiveAction;
if (!OtherHand->ActiveAction->bExtendable) return false;
FName ActionName;
for (const FExtendedGestureBinding& Gesture : OtherAction->Binding.ExtendedGestures)
{
if (Gesture.GestureName == Hand->ActiveGestureName)
{
ActionName = Gesture.GestureAction;
break;
}
}
if (ActionName.IsNone()) return false;
AGestureAction* NewAction = SpawnGestureAction(ActionName, Hand);
NewAction->InitAction(Hand, OtherHand->ActiveAction->Binding, UEUnitsPerUnit);
OtherHand->bUsedThisTick = true;
Hand->bUsedThisTick = true;
OtherAction->AddExtension(NewAction);
NewAction->AddExtension(OtherHand->ActiveAction);
return true;
}
+11 -5
View File
@@ -4,6 +4,7 @@
#include "HandManager.h"
#include "FingerTip.h"
#include "DrawDebugHelpers.h"
#include "GestureAction.h"
static const int32 HandBones[][2] = {
@@ -31,14 +32,21 @@ AHandActor::AHandActor()
}
void AHandActor::ResetGesture()
{
Phase = EHandGesturePhase::None;
ActiveGestureName = "";
CandidateFrames = 0;
FailFrames = 0;
if (IsValid(ActiveAction)) ActiveAction->EndGesture();
ActiveAction = nullptr;
}
// Called when the game starts or when spawned
void AHandActor::BeginPlay()
{
Super::BeginPlay();
GetWorld()->GetTimerManager().SetTimer(DeathTimer, this, &AHandActor::HandDeath, DeathTime, true);
}
// Called every frame
@@ -123,8 +131,6 @@ void AHandActor::UpdateHand(const FHandPacketHand& NewHand)
Confidence = NewHand.Confidence;
HandId = NewHand.HandId;
//UE_LOG(LogTemp, Error, TEXT("Update Hand"));
for (int32 n = 0; n < 5; n++)
{
FVector f = Points[FingerTipBones[n]];
+4 -2
View File
@@ -10,7 +10,7 @@ AHandManager::AHandManager()
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = true;
GestureManager = CreateDefaultSubobject<UGestureManagerComponent>(TEXT("GestureManager"));
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
}
// Called when the game starts or when spawned
@@ -18,6 +18,8 @@ void AHandManager::BeginPlay()
{
Super::BeginPlay();
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
if (!IsValid(HandActorClass))
{
HandActorClass = AHandActor::StaticClass();
@@ -69,7 +71,7 @@ void AHandManager::AddDeadHand(AHandActor* DeadHand)
DeadHands.Add(DeadHand);
}
TArray<AHandActor*> AHandManager::GetHandList()
TArray<AHandActor*>& AHandManager::GetHandList()
{
return HandList;
}
@@ -0,0 +1,83 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "MyGestureActions.h"
#include "HandActor.h"
#include "TableIconActor.h"
void ARulerAction::UpdateAction_Implementation()
{
if (!HandA) return;
// Update end point to current fingertip
Anchors.Add("End", HandA->Points[8]);
FVector Start = Anchors["Start"];
FVector End = Anchors["End"];
float RawDist = FVector::Dist(Start, End);
float Inches = GetSnappedInches(RawDist);
// Move line icon between start and end
if (Icons.IsValidIndex(0) && IsValid(Icons[0]))
{
Icons[0]->SetActorLocation((Start + End) * 0.5f);
// You'll add a SetEndpoints(Start, End) function to ATableIconLineActor later
}
// Move label icon to midpoint, slightly above table
if (Icons.IsValidIndex(1) && IsValid(Icons[1]))
{
FVector Mid = (Start + End) * 0.5f + FVector(0, 0, 5.f);
Icons[1]->SetActorLocation(Mid);
// Icons[1]->SetLabelText(FString::Printf(TEXT("%d\""), (int32)Inches));
}
}
void ARulerAction::AbortAction_Implementation()
{
Super::AbortAction();
}
void ARulerAction::AddExtension_Implementation(AGestureAction* OtherAction)
{
return;
}
void ARulerAction::EndGesture_Implementation()
{
Super::EndGesture();
}
void ARulerAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
{
Super::InitAction(Hand, NewBinding, NewUEUnitsPerUnit);
Anchors.Add("Start", Hand->Points[8]);
FActorSpawnParameters SpawnParams;
SpawnParams.Owner = GetOwner();
SpawnParams.SpawnCollisionHandlingOverride =
ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
AGestureAction* NewAction = nullptr;
/*AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
*ActionClassPtr,
FVector::ZeroVector,
FRotator::ZeroRotator,
SpawnParams);
*/
if (!NewAction) return;
}
float ARulerAction::GetSnappedInches(float RawDistance) const
{
if (UEUnitsPerUnit <= 0.f) return 0.f;
float Unit = RawDistance / UEUnitsPerUnit;
return FMath::RoundToFloat(Unit); // snap to whole inches
}
@@ -0,0 +1,167 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "MyTableIconActors.h"
#include "KismetProceduralMeshLibrary.h"
#include "TableIconActor.h"
#include "Kismet/GameplayStatics.h"
#include "Camera/CameraActor.h"
ATableIconLineActor::ATableIconLineActor()
{
MeshComponent = CreateDefaultSubobject<UProceduralMeshComponent>(TEXT("LineMesh"));
MeshComponent->SetupAttachment(RootComponent);
MeshComponent->SetCastShadow(false);
}
void ATableIconLineActor::SetEndpoints(const FVector& Start, const FVector& End)
{
// Skip rebuild if nothing meaningful changed
if (Start.Equals(LastStart, 0.1f) && End.Equals(LastEnd, 0.1f)) return;
LastStart = Start;
LastEnd = End;
BuildLineMesh(Start, End);
if (bShowTickMarks) BuildTickMesh(Start, End);
}
void ATableIconLineActor::BuildLineMesh(const FVector& Start, const FVector& End)
{
FVector Dir = (End - Start).GetSafeNormal();
FVector Up = FVector::UpVector;
FVector Right = FVector::CrossProduct(Dir, Up).GetSafeNormal() * LineThickness * 0.5f;
// A flat quad along the line, facing up
TArray<FVector> Vertices;
TArray<int32> Triangles;
TArray<FVector> Normals;
TArray<FVector2D> UVs;
TArray<FColor> Colors;
TArray<FProcMeshTangent> Tangents;
Vertices.Add(Start - Right); // 0
Vertices.Add(Start + Right); // 1
Vertices.Add(End + Right); // 2
Vertices.Add(End - Right); // 3
Triangles = { 0, 1, 2, 0, 2, 3 };
for (int i = 0; i < 4; i++)
{
Normals.Add(FVector::UpVector);
UVs.Add(FVector2D(i < 2 ? 0.f : 1.f, i % 2 == 0 ? 0.f : 1.f));
Colors.Add(LineColor.ToFColor(true));
Tangents.Add(FProcMeshTangent(Dir, false));
}
// Section 0 = main line
MeshComponent->CreateMeshSection(0, Vertices, Triangles, Normals, UVs,
Colors, Tangents, false);
}
void ATableIconLineActor::BuildTickMesh(const FVector& Start, const FVector& End)
{
if (UEUnitsPerUnit <= 0.f || TickInterval <= 0.f) return;
float TotalDist = FVector::Dist(Start, End);
float TickSpacing = TickInterval * UEUnitsPerUnit;
int32 NumTicks = FMath::FloorToInt(TotalDist / TickSpacing);
if (NumTicks <= 0)
{
MeshComponent->ClearMeshSection(1);
return;
}
FVector Dir = (End - Start).GetSafeNormal();
FVector Right = FVector::CrossProduct(Dir, FVector::UpVector).GetSafeNormal();
TArray<FVector> Vertices;
TArray<int32> Triangles;
TArray<FVector> Normals;
TArray<FVector2D> UVs;
TArray<FColor> Colors;
TArray<FProcMeshTangent> Tangents;
float TickHalfLen = LineThickness * 3.f;
float TickHalfW = LineThickness * 0.5f;
for (int32 i = 1; i <= NumTicks; i++)
{
FVector Center = Start + Dir * (TickSpacing * i);
FVector A = Center - Right * TickHalfLen - Dir * TickHalfW;
FVector B = Center + Right * TickHalfLen - Dir * TickHalfW;
FVector C = Center + Right * TickHalfLen + Dir * TickHalfW;
FVector D = Center - Right * TickHalfLen + Dir * TickHalfW;
int32 Base = Vertices.Num();
Vertices.Add(A); Vertices.Add(B);
Vertices.Add(C); Vertices.Add(D);
Triangles.Add(Base); Triangles.Add(Base + 1); Triangles.Add(Base + 2);
Triangles.Add(Base); Triangles.Add(Base + 2); Triangles.Add(Base + 3);
for (int j = 0; j < 4; j++)
{
Normals.Add(FVector::UpVector);
UVs.Add(FVector2D::ZeroVector);
Colors.Add(LineColor.ToFColor(true));
Tangents.Add(FProcMeshTangent(Dir, false));
}
}
// Section 1 = tick marks
MeshComponent->CreateMeshSection(1, Vertices, Triangles, Normals, UVs,
Colors, Tangents, false);
}
ATableIconWidgetActor::ATableIconWidgetActor()
{
PrimaryActorTick.bCanEverTick = true;
WidgetComponent = CreateDefaultSubobject<UWidgetComponent>(TEXT("WidgetComponent"));
WidgetComponent->SetupAttachment(RootComponent);
WidgetComponent->SetDrawAtDesiredSize(true);
WidgetComponent->SetWidgetSpace(EWidgetSpace::World);
WidgetComponent->SetCollisionEnabled(ECollisionEnabled::NoCollision);
WidgetInteraction = CreateDefaultSubobject<UWidgetInteractionComponent>(TEXT("WidgetInteraction"));
WidgetInteraction->SetupAttachment(RootComponent);
WidgetInteraction->InteractionSource = EWidgetInteractionSource::Custom;
WidgetInteraction->InteractionDistance = 10.f;
}
void ATableIconWidgetActor::Tick(float DeltaTime)
{
Super::Tick(DeltaTime);
if (!bFaceCamera) return;
APlayerCameraManager* CamManager = UGameplayStatics::GetPlayerCameraManager(this, 0);
if (!CamManager) return;
FVector CamLocation = CamManager->GetCameraLocation();
FVector ToCamera = (CamLocation - GetActorLocation()).GetSafeNormal();
// Only rotate on yaw — keeps widget flat relative to table
ToCamera.Z = 0.f;
if (!ToCamera.IsNearlyZero())
SetActorRotation(ToCamera.ToOrientationRotator());
}
void ATableIconWidgetActor::SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass)
{
if (!InWidgetClass) return;
WidgetComponent->SetWidgetClass(InWidgetClass);
WidgetComponent->InitWidget();
}
UUserWidget* ATableIconWidgetActor::GetWidget() const
{
return WidgetComponent->GetUserWidgetObject();
}
@@ -0,0 +1,30 @@
// Fill out your copyright notice in the Description page of Project Settings.
#include "TableIconActor.h"
// Sets default values
ATableIconActor::ATableIconActor()
{
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
PrimaryActorTick.bCanEverTick = false;
}
// Called when the game starts or when spawned
void ATableIconActor::BeginPlay()
{
Super::BeginPlay();
}
void ATableIconActor::OnActionEnded_Implementation()
{
if (!bPersistOnActionEnd) Destroy();
}
void ATableIconActor::OnActionUpdated_Implementation()
{
}
+2 -2
View File
@@ -107,8 +107,8 @@ FContentSource UUserHUDWidget::ParseContentSource(TSharedPtr<FJsonObject> Source
if (ContentTypeString == TEXT("Image")) Source.ContentType = EContentSourceType::Image;
else if (ContentTypeString == TEXT("PDF")) Source.ContentType = EContentSourceType::PDF;
else if (ContentTypeString == TEXT("TextFile")) Source.ContentType = EContentSourceType::TextFile;
else if (ContentTypeString == TEXT("InlineText")) Source.ContentType = EContentSourceType::InlineText;
else if (ContentTypeString == TEXT("TextFile")) Source.ContentType = EContentSourceType::TextFile;
else if (ContentTypeString == TEXT("InlineText")) Source.ContentType = EContentSourceType::InlineText;
else Source.ContentType = EContentSourceType::InlineText;
return Source;
+37 -7
View File
@@ -4,9 +4,12 @@
#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include "GestureData.h"
#include "GestureAction.generated.h"
class AHandActor;
class ATableIconActor;
class AHandManager;
UCLASS()
class TABLE_API AGestureAction : public AActor
@@ -22,8 +25,21 @@ protected:
virtual void BeginPlay() override;
public:
// Called every frame
virtual void Tick(float DeltaTime) override;
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
float UEUnitsPerUnit = 100.f;
UPROPERTY(EditDefaultsOnly)
bool bRequiresSecondHand = false;
UPROPERTY(EditDefaultsOnly)
bool bExtendable = false;
UPROPERTY(EditDefaultsOnly)
FGestureBinding Binding;
UPROPERTY(BlueprintReadWrite)
TArray<ATableIconActor*> Icons;
UPROPERTY(BlueprintReadWrite)
TMap<FName, FVector> Anchors;
@@ -34,15 +50,29 @@ public:
UPROPERTY(BlueprintReadWrite)
AHandActor* HandB;
UFUNCTION(BlueprintCallable)
UPROPERTY(BlueprintReadWrite)
AGestureAction* LinkedActions;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Stuff")
AHandManager* HandManager;
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void InitAction(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit);
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void UpdateAction();
UFUNCTION(BlueprintCallable)
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void AbortAction();
UFUNCTION(BlueprintCallable)
void AddExtension(AHandActor* OtherHand);
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void AddExtension(AGestureAction* OtherAction);
UFUNCTION(BlueprintCallable)
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void AddOtherHand(AHandActor* OtherHand);
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
void EndGesture();
void DestroyIcons();
};
+7 -7
View File
@@ -19,7 +19,7 @@ enum class EHandedness : uint8
};
UENUM(BlueprintType)
enum class EGesturePhase : uint8
enum class EHandGesturePhase : uint8
{
None,
Candidate,
@@ -54,10 +54,10 @@ struct FExtendedGestureBinding
GENERATED_BODY()
UPROPERTY(BlueprintReadOnly)
FString GestureName;
FName GestureName;
UPROPERTY(BlueprintReadOnly)
FString GestureAction;
FName GestureAction;
};
// What gets parsed from the JSON
@@ -67,15 +67,15 @@ struct FGestureBinding
GENERATED_BODY()
UPROPERTY(BlueprintReadOnly)
FString ActionName;
FName ActionName;
UPROPERTY(BlueprintReadOnly)
FString LeftGesture;
FName LeftGesture;
UPROPERTY(BlueprintReadOnly)
FString RightGesture;
FName RightGesture;
UPROPERTY(BlueprintReadOnly)
TArray<FExtendedGestureBinding> ExentdedGestures;
TArray<FExtendedGestureBinding> ExtendedGestures;
};
+85 -3
View File
@@ -31,8 +31,7 @@ class TABLE_API UGestureDefinition : public UObject
public:
UPROPERTY(BlueprintReadWrite)
FString GestureName;
FName GestureName;
// This checks the gesture and returns an aproximate scor of how well the Gesture was exectuded, needs to be overwritten in child classes.
UFUNCTION(BlueprintCallable)
@@ -50,13 +49,96 @@ protected:
float GetFingersAverage(const AHandActor* Hand, const TArray<EHandFinger>& Fingers, bool bExtended) const;
FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const;
const FVector& GetPoint(const AHandActor* Hand, int32 Index) const;
bool IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold = 0.7f) const;
};
UCLASS()
class TABLE_API UGesture_PinchPoint : public UGestureDefinition
class TABLE_API UGesture_PinchIndex : public UGestureDefinition
{
GENERATED_BODY()
public:
virtual float Evaluate(AHandActor* Hand) const override;
virtual void PostInitProperties() override
{
Super::PostInitProperties();
if (!HasAnyFlags(RF_ClassDefaultObject))
{
GestureName = TEXT("PinchIndex");
}
}
};
UCLASS()
class TABLE_API UGesture_PinchMiddleIndex : public UGestureDefinition
{
GENERATED_BODY()
public:
virtual float Evaluate(AHandActor* Hand) const override;
virtual void PostInitProperties() override
{
Super::PostInitProperties();
if (!HasAnyFlags(RF_ClassDefaultObject))
{
GestureName = FName("PinchMiddle");
}
}
};
UCLASS()
class TABLE_API UGesture_PointIndex : public UGestureDefinition
{
GENERATED_BODY()
public:
virtual float Evaluate(AHandActor* Hand) const override;
virtual void PostInitProperties() override
{
Super::PostInitProperties();
if (!HasAnyFlags(RF_ClassDefaultObject))
{
GestureName = FName("PointIndex");
}
}
};
UCLASS()
class TABLE_API UGesture_PointMiddle : public UGestureDefinition
{
GENERATED_BODY()
public:
virtual float Evaluate(AHandActor* Hand) const override;
virtual void PostInitProperties() override
{
Super::PostInitProperties();
if (!HasAnyFlags(RF_ClassDefaultObject))
{
GestureName = TEXT("PointMiddle");
}
}
};
UCLASS()
class TABLE_API UGesture_PointIndexMiddle : public UGestureDefinition
{
GENERATED_BODY()
public:
virtual float Evaluate(AHandActor* Hand) const override;
virtual void PostInitProperties() override
{
Super::PostInitProperties();
if (!HasAnyFlags(RF_ClassDefaultObject))
{
GestureName = TEXT("PointIndexMiddle");
}
}
};
@@ -7,7 +7,7 @@
#include "GestureData.h"
#include "GestureManagerComponent.generated.h"
class AHandManager;
UCLASS(ClassGroup = "Gesture", BlueprintType, Blueprintable, ClassGroup=(Custom), meta=(BlueprintSpawnableComponent) )
class TABLE_API UGestureManagerComponent : public UActorComponent
@@ -24,8 +24,11 @@ protected:
public:
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale")
float UEUnitsPerUnit = 100.f; // set via calibration rectangle
UPROPERTY(BlueprintReadOnly)
class AHandManager* HandManager;
AHandManager* HandManager;
UFUNCTION(BlueprintCallable, Category = "Gesture")
void EvaluateHands();
@@ -57,13 +60,13 @@ public:
// Populated on BeginPlay from the tables
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
TMap<FString, UGestureDefinition*> DefinitionMap;
TMap<FName, UGestureDefinition*> DefinitionMap;
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
TMap<FString, TSubclassOf<AGestureAction>> ActionClassMap;
TMap<FName, TSubclassOf<AGestureAction>> ActionClassMap;
private:
// GestureManager.h — add these
//
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
UDataTable* GestureDefinitionTable;
@@ -73,12 +76,13 @@ private:
void CheckGestureDefinitions(TArray<AHandActor*>& Hands);
void TriggerAction(AHandActor* Hand);
void UpdateOngoingAction();
void EndGesture();
void UpdateOngoingActions();
void SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B);
void ExtendGestureAction(FString ActionName, AHandActor* Hand);
void SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand);
AGestureAction* SpawnGestureAction(FName ActionName, AHandActor* Hand);
bool TryExtendGestureAction(AHandActor* Hand);
bool TryJoinExistingAction(AHandActor* Hand);
bool TryCreateExtension(AHandActor* Hand);
static constexpr int32 CandidateThreshold = 8;
static constexpr int32 FailThreshold = 5;
+7 -11
View File
@@ -11,6 +11,7 @@
class AHandManager;
class AFingerTip;
class AGestureAction;
UCLASS()
class TABLE_API AHandActor : public AActor
@@ -47,10 +48,13 @@ public:
// Gestures
UPROPERTY(BlueprintReadWrite)
EGesturePhase Phase = EGesturePhase::None;
EHandGesturePhase Phase = EHandGesturePhase::None;
UPROPERTY(BlueprintReadWrite)
FString ActiveGestureName;
FName ActiveGestureName;
UPROPERTY(BlueprintReadWrite)
AGestureAction* ActiveAction;
UPROPERTY(BlueprintReadWrite)
TMap<FString, FVector> Anchors;
@@ -67,15 +71,7 @@ public:
bool bUsedThisTick = false;
bool IsPaired() const { return PairedHand != nullptr; }
void ResetGesture()
{
Phase = EGesturePhase::None;
ActiveGestureName = "";
CandidateFrames = 0;
FailFrames = 0;
}
void ResetGesture();
protected:
// Called when the game starts or when spawned
+4 -3
View File
@@ -62,7 +62,7 @@ public:
FOnHandRemoved OnHandRemoved;
UFUNCTION(BlueprintCallable)
TArray<AHandActor*> GetHandList();
TArray<AHandActor*>& GetHandList();
//Gestures
@@ -72,12 +72,13 @@ public:
const TArray<AHandActor*>& GetHandList() const { return HandList; }
// GestureManager.h — add these
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture")
UDataTable* GestureDefinitionTable;
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture")
UDataTable* GestureActionTable;
private:
void RemoveDeadHands();
void UpdateHands();
@@ -0,0 +1,36 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "GestureAction.h"
#include "MyGestureActions.generated.h"
/**
*
*/
UCLASS()
class TABLE_API AMyGestureActions : public AGestureAction
{
GENERATED_BODY()
};
UCLASS()
class TABLE_API ARulerAction : public AGestureAction
{
GENERATED_BODY()
public:
void UpdateAction_Implementation() override;
void AbortAction_Implementation() override;
void AddExtension_Implementation(AGestureAction* OtherAction) override;
void EndGesture_Implementation() override;
void InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit) override;
private:
float GetSnappedInches(float RawDistance) const;
};
@@ -0,0 +1,93 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "TableIconActor.h"
#include "ProceduralMeshComponent.h"
#include "Components/WidgetComponent.h"
#include "Components/WidgetInteractionComponent.h"
#include "MyTableIconActors.generated.h"
class UWidgetComponent;
class UWidgetInteractionComponent;
/**
*
*/
UCLASS()
class TABLE_API AMyTableIconActors : public ATableIconActor
{
GENERATED_BODY()
};
UCLASS(Blueprintable)
class TABLE_API ATableIconLineActor : public ATableIconActor
{
GENERATED_BODY()
public:
ATableIconLineActor();
UFUNCTION(BlueprintCallable)
void SetEndpoints(const FVector& Start, const FVector& End);
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
float LineThickness = 1.f;
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
FLinearColor LineColor = FLinearColor::White;
// Tick markers along the line — e.g. every inch
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
bool bShowTickMarks = true;
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
float TickInterval = 1.f; // in real-world units
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
float UEUnitsPerUnit = 100.f;
private:
UPROPERTY()
UProceduralMeshComponent* MeshComponent;
void BuildLineMesh(const FVector& Start, const FVector& End);
void BuildTickMesh(const FVector& Start, const FVector& End);
FVector LastStart;
FVector LastEnd;
};
UCLASS(Blueprintable)
class TABLE_API ATableIconWidgetActor : public ATableIconActor
{
GENERATED_BODY()
public:
ATableIconWidgetActor();
UPROPERTY(VisibleAnywhere, BlueprintReadOnly)
UWidgetComponent* WidgetComponent;
UPROPERTY(VisibleAnywhere, BlueprintReadOnly)
UWidgetInteractionComponent* WidgetInteraction;
// Face toward a target each tick — useful for always facing the camera
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
bool bFaceCamera = true;
// Set widget content — callable from GestureAction
UFUNCTION(BlueprintCallable)
void SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass);
// Get the underlying widget cast to a specific type in BP
UFUNCTION(BlueprintCallable, BlueprintPure)
UUserWidget* GetWidget() const;
protected:
virtual void Tick(float DeltaTime) override;
};
@@ -0,0 +1,36 @@
// Fill out your copyright notice in the Description page of Project Settings.
#pragma once
#include "CoreMinimal.h"
#include "GameFramework/Actor.h"
#include "TableIconActor.generated.h"
UCLASS()
class TABLE_API ATableIconActor : public AActor
{
GENERATED_BODY()
public:
// Sets default values for this actor's properties
ATableIconActor();
protected:
// Called when the game starts or when spawned
virtual void BeginPlay() override;
public:
// If true, GestureAction will not destroy this on EndGesture
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
bool bPersistOnActionEnd = false;
// Called by GestureAction on end — BP overrides this for fade/lock behavior
UFUNCTION(BlueprintNativeEvent)
void OnActionEnded();
// Called by GestureAction every tick
UFUNCTION(BlueprintNativeEvent)
void OnActionUpdated();
};
+2 -1
View File
@@ -20,7 +20,8 @@ public class Table : ModuleRules
"UMG" ,
"Json",
"JsonUtilities",
"ImageWrapper"
"ImageWrapper",
"ProceduralMeshComponent"
});
PrivateDependencyModuleNames.AddRange(new string[] { });
+2 -2
View File
@@ -8,8 +8,8 @@ public class TableEditorTarget : TargetRules
public TableEditorTarget( TargetInfo Target) : base(Target)
{
Type = TargetType.Editor;
DefaultBuildSettings = BuildSettingsVersion.V6;
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
DefaultBuildSettings = BuildSettingsVersion.V7;
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
ExtraModuleNames.Add("Table");
}
}
+44
View File
@@ -0,0 +1,44 @@
<Solution>
<Configurations>
<BuildType Name="DebugGame" />
<BuildType Name="DebugGame Editor" />
<BuildType Name="Development" />
<BuildType Name="Development Editor" />
<BuildType Name="Shipping" />
<Platform Name="Win64" />
<Platform Name="Win64-arm64" />
<Platform Name="Win64-arm64ec" />
</Configurations>
<Folder Name="/Engine/">
<Project Path="Intermediate/ProjectFiles/UE5.vcxproj" Id="99002b2b-f5da-337f-af6f-e86d1d78a0d0">
<BuildType Project="BuiltWithUnrealBuildTool" />
<Platform Project="Win64" />
<Build Project="false" />
</Project>
</Folder>
<Folder Name="/Games/">
<Project Path="Intermediate/ProjectFiles/Table.vcxproj" Id="111ae310-8395-3d64-9119-33389f569f2d">
<BuildType Solution="DebugGame Editor|Win64" Project="DebugGame_Editor" />
<BuildType Solution="DebugGame Editor|Win64-arm64" Project="Win64_arm64_DebugGame_Editor" />
<BuildType Solution="DebugGame Editor|Win64-arm64ec" Project="Win64_arm64ec_DebugGame_Editor" />
<BuildType Solution="DebugGame|Win64-arm64" Project="Win64_arm64_DebugGame" />
<BuildType Solution="DebugGame|Win64-arm64ec" Project="Win64_arm64ec_DebugGame" />
<BuildType Solution="Development Editor|Win64" Project="Development_Editor" />
<BuildType Solution="Development Editor|Win64-arm64" Project="Win64_arm64_Development_Editor" />
<BuildType Solution="Development Editor|Win64-arm64ec" Project="Win64_arm64ec_Development_Editor" />
<BuildType Solution="Development|Win64-arm64" Project="Win64_arm64_Development" />
<BuildType Solution="Development|Win64-arm64ec" Project="Win64_arm64ec_Development" />
<BuildType Solution="Shipping|Win64-arm64" Project="Win64_arm64_Shipping" />
<BuildType Solution="Shipping|Win64-arm64ec" Project="Win64_arm64ec_Shipping" />
<Platform Project="x64" />
</Project>
</Folder>
<Folder Name="/Visualizers/">
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natjmc" />
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natstepfilter" />
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natvis" />
</Folder>
<Properties Name="ddbf523f-7eb6-4887-bd51-85a714ff87eb">
<Property Name="AvailablePlatforms" Value="Win64" />
</Properties>
</Solution>
+2 -2
View File
@@ -1,6 +1,6 @@
{
"FileVersion": 3,
"EngineAssociation": "5.7",
"EngineAssociation": "5.8",
"Category": "",
"Description": "",
"Modules": [
@@ -22,7 +22,7 @@
},
{
"Name": "VisualStudioTools",
"Enabled": true,
"Enabled": false,
"SupportedTargetPlatforms": [
"Win64"
],