Compare commits
5 Commits
d53d270dee
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
| c9a5c2c6ca | |||
| 85ab894665 | |||
| 257f045515 | |||
| c9e46eeb41 | |||
| aed58897cf |
@@ -4,8 +4,6 @@
|
|||||||
"Component.Unreal.Debugger",
|
"Component.Unreal.Debugger",
|
||||||
"Component.Unreal.Ide",
|
"Component.Unreal.Ide",
|
||||||
"Microsoft.Net.Component.4.6.2.TargetingPack",
|
"Microsoft.Net.Component.4.6.2.TargetingPack",
|
||||||
"Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL",
|
|
||||||
"Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64",
|
|
||||||
"Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL",
|
"Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL",
|
||||||
"Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64",
|
"Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64",
|
||||||
"Microsoft.VisualStudio.Component.VC.Llvm.Clang",
|
"Microsoft.VisualStudio.Component.VC.Llvm.Clang",
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
<Solution>
|
||||||
|
<Configurations>
|
||||||
|
<BuildType Name="Analyze" />
|
||||||
|
<BuildType Name="Debug" />
|
||||||
|
<BuildType Name="Development" />
|
||||||
|
<BuildType Name="Release" />
|
||||||
|
</Configurations>
|
||||||
|
<Folder Name="/Dependencies/">
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Analytics.Telemetry/EpicGames.Analytics.Telemetry.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build/EpicGames.Build.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph/EpicGames.BuildGraph.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core/EpicGames.Core.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde/EpicGames.Horde.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash/EpicGames.IoHash.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers/EpicGames.LogMatchers.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.MsBuild/EpicGames.MsBuild.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.OIDC/EpicGames.OIDC.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Oodle/EpicGames.Oodle.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Fixture/EpicGames.Perforce.Fixture.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed/EpicGames.Perforce.Managed.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce/EpicGames.Perforce.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.ProjectStore/EpicGames.ProjectStore.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization/EpicGames.Serialization.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UBA/EpicGames.UBA.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UHT/EpicGames.UHT.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/UnrealBuildTool/UnrealBuildTool.csproj" />
|
||||||
|
</Folder>
|
||||||
|
<Folder Name="/Tests/">
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build.Tests/EpicGames.Build.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph.Tests/EpicGames.BuildGraph.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core.Tests/EpicGames.Core.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde.Tests/EpicGames.Horde.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash.Tests/EpicGames.IoHash.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers.Tests/EpicGames.LogMatchers.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed.Tests/EpicGames.Perforce.Managed.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Tests/EpicGames.Perforce.Tests.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization.Tests/EpicGames.Serialization.Tests.csproj" />
|
||||||
|
</Folder>
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Platforms/VisionOS/Source/Programs/AutomationTool/VisionOS.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Android/Android.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Apple/Apple.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationTool.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationUtils/AutomationUtils.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/BuildGraph/BuildGraph.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CookedEditor/CookedEditor.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CrowdinLocalization/CrowdinLocalization.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Gauntlet/Gauntlet.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Insights/Insights.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/IOS/IOS.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Linux/Linux.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LiveLinkHub/LiveLinkHub.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Localization/Localization.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LowLevelTests/LowLevelTests.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mac/Mac.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Microsoft/Microsoft.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mutable/Mutable.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/OneSkyLocalization/OneSkyLocalization.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Scripts/AutomationScripts.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SmartlingLocalization/SmartlingLocalization.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SteamDeck/SteamDeck.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Turnkey/Turnkey.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/TVOS/TVOS.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Win/Win.Automation.csproj" />
|
||||||
|
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/XLocLocalization/XLocLocalization.Automation.csproj" />
|
||||||
|
</Solution>
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,68 @@
|
|||||||
|
# Table-UE
|
||||||
|
|
||||||
|
> **Work in progress.** This is an actively developed personal project, not a finished product — expect incomplete features, rough edges, and breaking changes.
|
||||||
|
|
||||||
|
Unreal Engine side of a real-time, gesture-controlled smart tabletop for tabletop wargaming (Warhammer 40k). A downward-facing multi-camera rig tracks players' hands over the physical table; this project receives the 3D hand data over UDP, recognizes gestures, and drives an interactive projected overlay (rulers, markers, HUDs, etc.) directly on the tabletop surface.
|
||||||
|
|
||||||
|
The companion computer-vision/UI application that captures and streams the hand-tracking data lives in [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python).
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
- Built with **Unreal Engine 5.8** (C++ gameplay module + Blueprints).
|
||||||
|
- Receives per-frame 3D hand landmark data (21 points/hand, MediaPipe topology) via a **raw UDP socket**, decoded into `FHandPacketHand` structs.
|
||||||
|
- Spawns and updates `AHandActor` instances per tracked hand and exposes them to Blueprints for visualization/debugging.
|
||||||
|
- A **JSON-driven gesture binding system** matches live hand poses against configurable gesture definitions (e.g. "point with index finger") and triggers corresponding **Gesture Actions**.
|
||||||
|
- Gesture Actions are self-contained, extendable Actors that own their own on-table visuals ("Table Icons") — e.g. a `Ruler` gesture action that draws a measuring line with tick marks between two points and displays the distance in real-world units.
|
||||||
|
- A calibration scale factor (`UEUnitsPerUnit`) converts between real-world measurements (inches/cm on the physical table) and Unreal units, set up per session.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
Table-Python (multi-camera capture, MediaPipe hand tracking, stereo triangulation)
|
||||||
|
│ UDP (custom binary hand packet protocol)
|
||||||
|
▼
|
||||||
|
ADetectionManager — UDP receiver, decodes packets into hand data
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
AHandManager — spawns/despawns AHandActor per tracked hand id
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
UGestureManagerComponent — loads gesture definitions from JSON, evaluates hand poses each tick
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
AGestureAction (+ subclasses, e.g. AGestureAction_Ruler)
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
ATableIconActor (+ subclasses) — the actual on-table visuals (lines, labels, procedural meshes, UMG widgets)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key classes
|
||||||
|
|
||||||
|
| Class | Responsibility |
|
||||||
|
|---|---|
|
||||||
|
| [`ADetectionManager`](Source/Table/Public/DetectionManager.h) | Owns the UDP socket, parses incoming hand-tracking packets |
|
||||||
|
| [`AHandManager`](Source/Table/Public/HandManager.h) | Tracks the set of currently-active hands, broadcasts added/removed delegates |
|
||||||
|
| [`AHandActor`](Source/Table/Public/HandActor.h) / [`FingerTip`](Source/Table/Public/FingerTip.h) | Represents a single tracked hand and its fingertip positions in world space |
|
||||||
|
| [`UGestureManagerComponent`](Source/Table/Public/GestureManagerComponent.h) | Loads a JSON gesture-binding config and evaluates gestures against tracked hands |
|
||||||
|
| [`UGestureDefinition`](Source/Table/Public/GestureDefinition.h) | Scores how well a hand pose matches a named gesture (overridden per gesture) |
|
||||||
|
| [`AGestureAction`](Source/Table/Public/GestureAction.h) / [`AGestureAction_Ruler`](Source/Table/Public/MyGestureActions.h) | Lifecycle (init/update/end) for an active gesture, e.g. a ruler measurement |
|
||||||
|
| [`ATableIconActor`](Source/Table/Public/TableIconActor.h) / [`ATableIconLineActor`](Source/Table/Public/MyTableIconActors.h) | Visual elements drawn on the table for the duration of a gesture |
|
||||||
|
|
||||||
|
Gesture-to-action bindings are configured in JSON (see `Content/WorldObjects/GestureActions/`), so new gestures can be wired up without touching C++.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- Unreal Engine **5.8**
|
||||||
|
- Visual Studio 2022 (Windows) with the "Game development with C++" workload
|
||||||
|
- A running instance of [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python) sending hand-tracking data over UDP (default `127.0.0.1:9000`) if you want live hand input; the level can otherwise be explored/tested without a live feed.
|
||||||
|
|
||||||
|
## Getting started
|
||||||
|
|
||||||
|
1. Clone the repo and generate project files by right-clicking `Table.uproject` → **Generate Visual Studio project files** (or open it directly in the Unreal Editor, which will prompt to build missing modules).
|
||||||
|
2. Open `Table.sln` in Visual Studio and build, or open `Table.uproject` in Unreal Editor.
|
||||||
|
3. Press **Play** to start the `ADetectionManager` UDP listener.
|
||||||
|
4. Start the Python tracking application to begin streaming live hand data to the table.
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
Personal/hobby project, actively evolving — gesture set, table icons, and HUD content are still being expanded.
|
||||||
@@ -8,8 +8,8 @@ public class TableTarget : TargetRules
|
|||||||
public TableTarget(TargetInfo Target) : base(Target)
|
public TableTarget(TargetInfo Target) : base(Target)
|
||||||
{
|
{
|
||||||
Type = TargetType.Game;
|
Type = TargetType.Game;
|
||||||
DefaultBuildSettings = BuildSettingsVersion.V6;
|
DefaultBuildSettings = BuildSettingsVersion.V7;
|
||||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
|
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
|
||||||
ExtraModuleNames.Add("Table");
|
ExtraModuleNames.Add("Table");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -197,8 +197,8 @@ bool ADetectionManager::ParseHandPacket(const TArray<uint8>& Bytes, int32& OutSe
|
|||||||
|
|
||||||
FHandPacketHand Hand;
|
FHandPacketHand Hand;
|
||||||
Hand.HandId = (int32)HandId;
|
Hand.HandId = (int32)HandId;
|
||||||
Hand.Handedness = (HandednessByte == 1) ? EHandedness::Left :
|
Hand.Handedness = (HandednessByte == 1) ? EHandedness::Right :
|
||||||
(HandednessByte == 2) ? EHandedness::Right :
|
(HandednessByte == 2) ? EHandedness::Left :
|
||||||
EHandedness::Unknown;
|
EHandedness::Unknown;
|
||||||
Hand.Confidence = Confidence;
|
Hand.Confidence = Confidence;
|
||||||
|
|
||||||
|
|||||||
@@ -3,44 +3,64 @@
|
|||||||
|
|
||||||
#include "GestureAction.h"
|
#include "GestureAction.h"
|
||||||
#include "HandActor.h"
|
#include "HandActor.h"
|
||||||
|
#include "TableIconActor.h"
|
||||||
|
|
||||||
// Sets default values
|
// Sets default values
|
||||||
AGestureAction::AGestureAction()
|
AGestureAction::AGestureAction()
|
||||||
{
|
{
|
||||||
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||||
PrimaryActorTick.bCanEverTick = true;
|
PrimaryActorTick.bCanEverTick = false;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called when the game starts or when spawned
|
// Called when the game starts or when spawned
|
||||||
void AGestureAction::BeginPlay()
|
void AGestureAction::BeginPlay()
|
||||||
{
|
{
|
||||||
Super::BeginPlay();
|
Super::BeginPlay();
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called every frame
|
void AGestureAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit, float TableHeight)
|
||||||
void AGestureAction::Tick(float DeltaTime)
|
|
||||||
{
|
{
|
||||||
Super::Tick(DeltaTime);
|
HandA = Hand;
|
||||||
|
bRequiresSecondHand = Binding.LeftGesture != "" && Binding.RightGesture != "";
|
||||||
|
bExtendable = Binding.ExtendedGestures.IsEmpty();
|
||||||
|
Binding = NewBinding;
|
||||||
|
HandManager = Hand->HandManager;
|
||||||
|
UEUnitsPerUnit = NewUEUnitsPerUnit;
|
||||||
}
|
}
|
||||||
|
|
||||||
void AGestureAction::UpdateAction()
|
void AGestureAction::UpdateAction_Implementation()
|
||||||
{
|
{
|
||||||
Destroy();
|
Destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
void AGestureAction::AbortAction()
|
void AGestureAction::AbortAction_Implementation()
|
||||||
{
|
{
|
||||||
|
for (ATableIconActor* Icon : Icons)
|
||||||
|
Icon->bPersistOnActionEnd = false;
|
||||||
|
EndGesture();
|
||||||
}
|
}
|
||||||
|
|
||||||
void AGestureAction::AddExtension(AHandActor* OtherHand)
|
void AGestureAction::AddExtension_Implementation(AGestureAction* OtherAction)
|
||||||
|
{
|
||||||
|
LinkedActions = OtherAction;
|
||||||
|
}
|
||||||
|
|
||||||
|
void AGestureAction::AddOtherHand_Implementation(AHandActor* OtherHand)
|
||||||
{
|
{
|
||||||
HandB = OtherHand;
|
HandB = OtherHand;
|
||||||
}
|
}
|
||||||
|
|
||||||
void AGestureAction::EndGesture()
|
void AGestureAction::EndGesture_Implementation()
|
||||||
{
|
{
|
||||||
|
HandA = nullptr;
|
||||||
|
HandB = nullptr;
|
||||||
|
DestroyIcons();
|
||||||
Destroy();
|
Destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void AGestureAction::DestroyIcons()
|
||||||
|
{
|
||||||
|
for (ATableIconActor* Icon : Icons)
|
||||||
|
if (IsValid(Icon) ) Icon->OnActionEnded();
|
||||||
|
Icons.Empty();
|
||||||
|
}
|
||||||
@@ -40,6 +40,11 @@ const FVector& UGestureDefinition::GetPoint(const AHandActor* Hand, int32 Index)
|
|||||||
return Hand->Points[Index];
|
return Hand->Points[Index];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool UGestureDefinition::IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold) const
|
||||||
|
{
|
||||||
|
return GetFingerExtendedScore(Hand, Finger) > Threshold;
|
||||||
|
}
|
||||||
|
|
||||||
float UGestureDefinition::GetHandScale(const AHandActor* Hand) const
|
float UGestureDefinition::GetHandScale(const AHandActor* Hand) const
|
||||||
{
|
{
|
||||||
const TArray<FVector>& P = Hand->Points;
|
const TArray<FVector>& P = Hand->Points;
|
||||||
@@ -53,30 +58,44 @@ float UGestureDefinition::GetNormalizedDistance(const AHandActor* Hand, int32 A,
|
|||||||
}
|
}
|
||||||
float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const
|
float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const
|
||||||
{
|
{
|
||||||
float Dist = GetNormalizedDistance(
|
const FVector& P1 = Hand->Points[GetFingerTipIndex(A)];
|
||||||
Hand,
|
const FVector& P2 = Hand->Points[GetFingerTipIndex(B)];
|
||||||
GetFingerTipIndex(A),
|
|
||||||
GetFingerTipIndex(B)
|
float Dist = FVector::Dist(P1, P2);
|
||||||
);
|
|
||||||
|
float HandScale = FVector::Dist( Hand->Points[0], Hand->Points[9] );
|
||||||
|
|
||||||
|
float Normalized = Dist / HandScale;
|
||||||
|
|
||||||
|
//UE_LOG(LogTemp, Warning, TEXT("Dist: %.3f | HandScale: %.3f | Norm: %.3f"), Dist, HandScale, Normalized);
|
||||||
|
|
||||||
const float Threshold = 0.25f;
|
const float Threshold = 0.25f;
|
||||||
|
|
||||||
return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f);
|
return 1.f - FMath::Clamp( Normalized / Threshold, 0.f, 1.f );
|
||||||
}
|
}
|
||||||
|
|
||||||
float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const
|
float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const
|
||||||
{
|
{
|
||||||
const TArray<FVector>& P = Hand->Points;
|
const TArray<FVector>& P = Hand->Points;
|
||||||
|
|
||||||
|
if (Finger == EHandFinger::Thumb)
|
||||||
|
{
|
||||||
|
// Thumb: compare tip->base against index base->wrist as reference direction
|
||||||
|
const FVector& Base = P[2];
|
||||||
|
const FVector& Tip = P[4];
|
||||||
|
const FVector& Ref = P[5]; // index base as stable reference
|
||||||
|
float Dot = FVector::DotProduct(
|
||||||
|
(Tip - Base).GetSafeNormal(),
|
||||||
|
(Ref - P[0]).GetSafeNormal());
|
||||||
|
return FMath::Clamp(Dot, 0.f, 1.f);
|
||||||
|
}
|
||||||
|
|
||||||
const FVector& Wrist = P[0];
|
const FVector& Wrist = P[0];
|
||||||
const FVector& Base = P[GetFingerBaseIndex(Finger)];
|
const FVector& Base = P[GetFingerBaseIndex(Finger)];
|
||||||
const FVector& Tip = P[GetFingerTipIndex(Finger)];
|
const FVector& Tip = P[GetFingerTipIndex(Finger)];
|
||||||
|
|
||||||
float Dot = FVector::DotProduct(
|
float Dot = FVector::DotProduct(
|
||||||
(Tip - Base).GetSafeNormal(),
|
(Tip - Base).GetSafeNormal(),
|
||||||
(Base - Wrist).GetSafeNormal()
|
(Base - Wrist).GetSafeNormal());
|
||||||
);
|
|
||||||
|
|
||||||
return FMath::Clamp(Dot, 0.f, 1.f);
|
return FMath::Clamp(Dot, 0.f, 1.f);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,9 +110,7 @@ float UGestureDefinition::GetFingersAverage(const AHandActor* Hand, const TArray
|
|||||||
|
|
||||||
for (EHandFinger Finger : Fingers)
|
for (EHandFinger Finger : Fingers)
|
||||||
{
|
{
|
||||||
Sum += bExtended
|
Sum += bExtended ? GetFingerExtendedScore(Hand, Finger) : GetFingerCurledScore(Hand, Finger);
|
||||||
? GetFingerExtendedScore(Hand, Finger)
|
|
||||||
: GetFingerCurledScore(Hand, Finger);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f;
|
return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f;
|
||||||
@@ -107,19 +124,57 @@ FVector UGestureDefinition::GetFingerDirection(const AHandActor* Hand, EHandFing
|
|||||||
return (Tip - Base).GetSafeNormal();
|
return (Tip - Base).GetSafeNormal();
|
||||||
}
|
}
|
||||||
|
|
||||||
float UGesture_PinchPoint::Evaluate(AHandActor* Hand) const
|
float UGesture_PinchIndex::Evaluate(AHandActor* Hand) const
|
||||||
{
|
{
|
||||||
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
|
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
|
||||||
float IndexExtended = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
float IndexExtended = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||||
|
|
||||||
float OthersCurled = GetFingersAverage(
|
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky }, false );
|
||||||
Hand,
|
|
||||||
{ EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky },
|
|
||||||
false
|
|
||||||
);
|
|
||||||
|
|
||||||
return
|
return Pinch * 0.5f + IndexExtended * 0.3f + OthersCurled * 0.2f;
|
||||||
Pinch * 0.5f +
|
}
|
||||||
IndexExtended * 0.3f +
|
|
||||||
OthersCurled * 0.2f;
|
float UGesture_PinchMiddleIndex::Evaluate(AHandActor* Hand) const
|
||||||
|
{
|
||||||
|
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Middle);
|
||||||
|
Pinch += GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
|
||||||
|
|
||||||
|
float MiddleExtended = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||||
|
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Ring, EHandFinger::Pinky }, false );
|
||||||
|
|
||||||
|
return Pinch * 0.5f + MiddleExtended * 0.3f + OthersCurled * 0.2f;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
float UGesture_PointIndex::Evaluate(AHandActor* Hand) const
|
||||||
|
{
|
||||||
|
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||||
|
float Middle = GetFingerCurledScore(Hand, EHandFinger::Middle);
|
||||||
|
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||||
|
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||||
|
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||||
|
|
||||||
|
return ( Index + Middle + Ring + Pinky + Thumb ) / 5.f;
|
||||||
|
}
|
||||||
|
|
||||||
|
float UGesture_PointMiddle::Evaluate(AHandActor* Hand) const
|
||||||
|
{
|
||||||
|
float Index = GetFingerCurledScore(Hand, EHandFinger::Index);
|
||||||
|
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||||
|
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||||
|
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||||
|
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||||
|
|
||||||
|
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
|
||||||
|
}
|
||||||
|
|
||||||
|
float UGesture_PointIndexMiddle::Evaluate(AHandActor* Hand) const
|
||||||
|
{
|
||||||
|
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||||
|
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||||
|
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||||
|
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||||
|
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||||
|
|
||||||
|
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ UGestureManagerComponent::UGestureManagerComponent()
|
|||||||
// off to improve performance if you don't need them.
|
// off to improve performance if you don't need them.
|
||||||
PrimaryComponentTick.bCanEverTick = false;
|
PrimaryComponentTick.bCanEverTick = false;
|
||||||
|
|
||||||
HandManager = Cast<AHandManager>(GetOwner());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -23,101 +23,14 @@ void UGestureManagerComponent::BeginPlay()
|
|||||||
{
|
{
|
||||||
Super::BeginPlay();
|
Super::BeginPlay();
|
||||||
|
|
||||||
|
HandManager = Cast<AHandManager>(GetOwner());
|
||||||
|
|
||||||
HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded);
|
HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded);
|
||||||
HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved);
|
HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved);
|
||||||
|
|
||||||
LoadBindings("C:/git/Table/Test/Gestures/gesture_config.json");
|
LoadBindings("C:/git/Table/Test/Gestures/gesture_config.json");
|
||||||
}
|
}
|
||||||
|
|
||||||
void UGestureManagerComponent::EvaluateHands()
|
|
||||||
{
|
|
||||||
TArray<AHandActor*>& Hands = HandManager->HandList;
|
|
||||||
|
|
||||||
for (AHandActor* Hand : Hands) Hand->bUsedThisTick = false;
|
|
||||||
|
|
||||||
CheckGestureDefinitions(Hands);
|
|
||||||
|
|
||||||
for (AHandActor* Hand : Hands)
|
|
||||||
{
|
|
||||||
if (Hand->bUsedThisTick) continue;
|
|
||||||
switch (Hand->Phase)
|
|
||||||
{
|
|
||||||
case EGesturePhase::Started:
|
|
||||||
TriggerAction(Hand);
|
|
||||||
break;
|
|
||||||
case EGesturePhase::Ended:
|
|
||||||
Hand->ResetGesture();
|
|
||||||
EndGesture();
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
UpdateOngoingAction();
|
|
||||||
}
|
|
||||||
|
|
||||||
void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
|
|
||||||
{
|
|
||||||
Hand->ResetGesture();
|
|
||||||
}
|
|
||||||
|
|
||||||
void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand)
|
|
||||||
{
|
|
||||||
Hand->PairedHand->PairedHand = nullptr;
|
|
||||||
Hand->ResetGesture();
|
|
||||||
}
|
|
||||||
|
|
||||||
void UGestureManagerComponent::LoadBindings(const FString& FilePath)
|
|
||||||
{
|
|
||||||
FString JsonString;
|
|
||||||
|
|
||||||
if (!FFileHelper::LoadFileToString(JsonString, *FilePath))
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
TSharedPtr<FJsonObject> Root;
|
|
||||||
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(JsonString);
|
|
||||||
|
|
||||||
if (!FJsonSerializer::Deserialize(Reader, Root) || !Root.IsValid())
|
|
||||||
return;
|
|
||||||
|
|
||||||
const TArray<TSharedPtr<FJsonValue>>* BindingsArray;
|
|
||||||
|
|
||||||
if (Root->TryGetArrayField(TEXT("bindings"), BindingsArray))
|
|
||||||
{
|
|
||||||
for (const TSharedPtr<FJsonValue>& Value : *BindingsArray)
|
|
||||||
{
|
|
||||||
TSharedPtr<FJsonObject> Obj = Value->AsObject();
|
|
||||||
if (!Obj.IsValid()) continue;
|
|
||||||
|
|
||||||
FGestureBinding Binding;
|
|
||||||
|
|
||||||
Obj->TryGetStringField(TEXT("action"), Binding.ActionName);
|
|
||||||
Obj->TryGetStringField(TEXT("left"), Binding.LeftGesture);
|
|
||||||
Obj->TryGetStringField(TEXT("right"), Binding.RightGesture);
|
|
||||||
|
|
||||||
const TArray<TSharedPtr<FJsonValue>>* ExtArray;
|
|
||||||
if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray))
|
|
||||||
{
|
|
||||||
for (const TSharedPtr<FJsonValue>& ExtVal : *ExtArray)
|
|
||||||
{
|
|
||||||
TSharedPtr<FJsonObject> ExtObj = ExtVal->AsObject();
|
|
||||||
if (!ExtObj.IsValid()) continue;
|
|
||||||
|
|
||||||
FExtendedGestureBinding Ext;
|
|
||||||
|
|
||||||
ExtObj->TryGetStringField(TEXT("gesture"), Ext.GestureName);
|
|
||||||
ExtObj->TryGetStringField(TEXT("action"), Ext.GestureAction);
|
|
||||||
|
|
||||||
Binding.ExentdedGestures.Add(Ext);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
GestureBindings.Add(Binding);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void UGestureManagerComponent::Initialize(UDataTable* InGestureTable, UDataTable* InActionTable)
|
void UGestureManagerComponent::Initialize(UDataTable* InGestureTable, UDataTable* InActionTable)
|
||||||
{
|
{
|
||||||
GestureDefinitionTable = InGestureTable;
|
GestureDefinitionTable = InGestureTable;
|
||||||
@@ -143,6 +56,7 @@ void UGestureManagerComponent::BuildDefinitionMap()
|
|||||||
UGestureDefinition* Def = NewObject<UGestureDefinition>(this, Row->DefinitionClass);
|
UGestureDefinition* Def = NewObject<UGestureDefinition>(this, Row->DefinitionClass);
|
||||||
|
|
||||||
DefinitionMap.Add(Def->GestureName, Def);
|
DefinitionMap.Add(Def->GestureName, Def);
|
||||||
|
GestureDefinitions.Add(Def);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,184 +66,285 @@ void UGestureManagerComponent::BuildActionMap()
|
|||||||
|
|
||||||
static const FString Context = TEXT("GestureActionContext");
|
static const FString Context = TEXT("GestureActionContext");
|
||||||
|
|
||||||
TArray<FGestureActionRow*> Rows;
|
TArray<FName> RowNames = GestureActionTable->GetRowNames();
|
||||||
GestureActionTable->GetAllRows(Context, Rows);
|
|
||||||
|
|
||||||
for (FGestureActionRow* Row : Rows)
|
for (const FName& RowName : RowNames)
|
||||||
{
|
{
|
||||||
|
FGestureActionRow* Row = GestureActionTable->FindRow<FGestureActionRow>( RowName, Context);
|
||||||
|
|
||||||
if (!Row || !Row->ActionClass) continue;
|
if (!Row || !Row->ActionClass) continue;
|
||||||
|
|
||||||
ActionClassMap.Add(
|
ActionClassMap.Add(RowName, Row->ActionClass);
|
||||||
Row->ActionClass->GetName(),
|
}
|
||||||
Row->ActionClass
|
|
||||||
|
|
||||||
);
|
}
|
||||||
|
|
||||||
|
void UGestureManagerComponent::EvaluateHands()
|
||||||
|
{
|
||||||
|
TArray<AHandActor*>& Hands = HandManager->HandList;
|
||||||
|
|
||||||
|
for (AHandActor* Hand : Hands) Hand->bUsedThisTick = false;
|
||||||
|
|
||||||
|
CheckGestureDefinitions(Hands);
|
||||||
|
|
||||||
|
for (AHandActor* Hand : Hands)
|
||||||
|
{
|
||||||
|
if (Hand->bUsedThisTick) continue;
|
||||||
|
switch (Hand->Phase)
|
||||||
|
{
|
||||||
|
case EHandGesturePhase::Started:
|
||||||
|
TriggerAction(Hand);
|
||||||
|
break;
|
||||||
|
case EHandGesturePhase::Ended:
|
||||||
|
Hand->ResetGesture();
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
UpdateOngoingActions();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
|
||||||
|
{
|
||||||
|
Hand->ResetGesture();
|
||||||
|
}
|
||||||
|
|
||||||
|
void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand)
|
||||||
|
{
|
||||||
|
if (!Hand->ActiveAction) return;
|
||||||
|
if (!ActiveGestureActions.Contains(Hand->ActiveAction)) return;
|
||||||
|
Hand->ActiveAction->Destroy();
|
||||||
|
ActiveGestureActions.Remove(Hand->ActiveAction);
|
||||||
|
}
|
||||||
|
|
||||||
|
void UGestureManagerComponent::LoadBindings(const FString& FilePath)
|
||||||
|
{
|
||||||
|
FString JsonString = "";
|
||||||
|
FString Temp = "";
|
||||||
|
if (!FFileHelper::LoadFileToString(JsonString, *FilePath))
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
TSharedPtr<FJsonObject> Root;
|
||||||
|
TSharedRef<TJsonReader<>> Reader = TJsonReaderFactory<>::Create(JsonString);
|
||||||
|
|
||||||
|
if (!FJsonSerializer::Deserialize(Reader, Root) || !Root.IsValid())
|
||||||
|
return;
|
||||||
|
|
||||||
|
const TArray<TSharedPtr<FJsonValue>>* BindingsArray;
|
||||||
|
|
||||||
|
if (Root->TryGetArrayField(TEXT("rules"), BindingsArray))
|
||||||
|
{
|
||||||
|
for (const TSharedPtr<FJsonValue>& Value : *BindingsArray)
|
||||||
|
{
|
||||||
|
TSharedPtr<FJsonObject> Obj = Value->AsObject();
|
||||||
|
if (!Obj.IsValid()) continue;
|
||||||
|
|
||||||
|
FGestureBinding Binding;
|
||||||
|
|
||||||
|
Obj->TryGetStringField(TEXT("action"), Temp);
|
||||||
|
Binding.ActionName = FName(Temp);
|
||||||
|
Obj->TryGetStringField(TEXT("left"), Temp);
|
||||||
|
Binding.LeftGesture = FName(Temp);
|
||||||
|
Obj->TryGetStringField(TEXT("right"), Temp);
|
||||||
|
Binding.RightGesture = FName(Temp);
|
||||||
|
|
||||||
|
const TArray<TSharedPtr<FJsonValue>>* ExtArray;
|
||||||
|
if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray))
|
||||||
|
{
|
||||||
|
for (const TSharedPtr<FJsonValue>& ExtVal : *ExtArray)
|
||||||
|
{
|
||||||
|
TSharedPtr<FJsonObject> ExtObj = ExtVal->AsObject();
|
||||||
|
if (!ExtObj.IsValid()) continue;
|
||||||
|
|
||||||
|
FExtendedGestureBinding Ext;
|
||||||
|
|
||||||
|
ExtObj->TryGetStringField(TEXT("gesture"), Temp);
|
||||||
|
Ext.GestureName = FName(Temp);
|
||||||
|
ExtObj->TryGetStringField(TEXT("action"), Temp);
|
||||||
|
Ext.GestureAction = FName(Temp);
|
||||||
|
|
||||||
|
Binding.ExtendedGestures.Add(Ext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
GestureBindings.Add(Binding);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
void UGestureManagerComponent::CheckGestureDefinitions(TArray<AHandActor*>& Hands)
|
void UGestureManagerComponent::CheckGestureDefinitions(TArray<AHandActor*>& Hands)
|
||||||
{
|
{
|
||||||
const float MinScoreThreshold = 0.5f;
|
//const float MinScoreThreshold = 0.7f;
|
||||||
|
|
||||||
|
|
||||||
for (AHandActor* Hand : Hands) {
|
for (AHandActor* Hand : Hands) {
|
||||||
|
float Threshold = (Hand->Phase == EHandGesturePhase::Ongoing || Hand->Phase == EHandGesturePhase::Started) ? 0.60f : 0.75f;
|
||||||
|
|
||||||
float BestScore = 0.f;
|
float BestScore = 0.f;
|
||||||
FString BestGesture = "";
|
FName BestGesture = "";
|
||||||
|
|
||||||
for (UGestureDefinition* Gesture : GestureDefinitions) {
|
for (UGestureDefinition* Gesture : GestureDefinitions) {
|
||||||
|
|
||||||
float Score = Gesture->Evaluate(Hand);
|
float Score = Gesture->Evaluate(Hand); // Getting the Score of the gestures to check
|
||||||
|
|
||||||
if (Score > BestScore)
|
//UE_LOG(LogTemp, Warning, TEXT("%s -> %s : %.3f"), *Hand->GetName(), *Gesture->GestureName, Score);
|
||||||
|
|
||||||
|
if (Score > BestScore) // Getting the gesture with the highest Score
|
||||||
{
|
{
|
||||||
BestScore = Score;
|
BestScore = Score;
|
||||||
BestGesture = Gesture->GestureName;
|
BestGesture = Gesture->GestureName;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (BestScore < MinScoreThreshold)
|
// Check if the gesture with the highest score is a valid Gesture
|
||||||
|
if (BestScore < Threshold)
|
||||||
{
|
{
|
||||||
if (Hand->Phase != EGesturePhase::None)
|
if (Hand->Phase != EHandGesturePhase::None)
|
||||||
{
|
{
|
||||||
Hand->FailFrames++;
|
Hand->FailFrames++;
|
||||||
|
UE_LOG(LogTemp, Warning,
|
||||||
|
TEXT("[%s] Below threshold (%s) Score=%.3f Fail=%d/%d"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*Hand->ActiveGestureName.ToString(),
|
||||||
|
BestScore,
|
||||||
|
Hand->FailFrames,
|
||||||
|
FailThreshold);
|
||||||
|
//if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
|
||||||
if (Hand->FailFrames >= FailThreshold)
|
if (Hand->FailFrames >= FailThreshold)
|
||||||
{
|
{
|
||||||
Hand->Phase = EGesturePhase::Ended;
|
UE_LOG(LogTemp, Warning,
|
||||||
}
|
TEXT("[%s] -> ENDED (%s)"),
|
||||||
}
|
*Hand->GetName(),
|
||||||
|
*Hand->ActiveGestureName.ToString());
|
||||||
|
|
||||||
|
Hand->Phase = EHandGesturePhase::Ended;
|
||||||
|
}
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Handle valid Gesture
|
||||||
switch (Hand->Phase)
|
switch (Hand->Phase)
|
||||||
{
|
{
|
||||||
case EGesturePhase::None:
|
case EHandGesturePhase::None: // Starting a Candidate
|
||||||
Hand->ActiveGestureName = BestGesture;
|
Hand->ActiveGestureName = BestGesture;
|
||||||
Hand->Phase = EGesturePhase::Candidate;
|
Hand->Phase = EHandGesturePhase::Candidate;
|
||||||
Hand->CandidateFrames = 1;
|
Hand->CandidateFrames = 1;
|
||||||
Hand->FailFrames = 0;
|
Hand->FailFrames = 0;
|
||||||
|
UE_LOG(LogTemp, Warning,
|
||||||
|
TEXT("[%s] NONE -> CANDIDATE (%s)"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*BestGesture.ToString());
|
||||||
|
|
||||||
break;
|
break;
|
||||||
case EGesturePhase::Candidate:
|
case EHandGesturePhase::Candidate: // Checking if the Candidate stays valid or another gesture is made
|
||||||
if (Hand->ActiveGestureName != BestGesture)
|
if (Hand->ActiveGestureName == BestGesture)
|
||||||
{
|
{
|
||||||
Hand->FailFrames++;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
Hand->CandidateFrames++;
|
Hand->CandidateFrames++;
|
||||||
Hand->FailFrames = 0;
|
Hand->FailFrames = 0;
|
||||||
|
|
||||||
if (Hand->CandidateFrames >= CandidateThreshold)
|
UE_LOG(LogTemp, Warning,
|
||||||
Hand->Phase = EGesturePhase::Started;
|
TEXT("[%s] Candidate %s (%d/%d)"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*BestGesture.ToString(),
|
||||||
|
Hand->CandidateFrames,
|
||||||
|
CandidateThreshold);
|
||||||
|
|
||||||
|
//if (Hand->CandidateFrames >= CandidateThreshold) Hand->Phase = EHandGesturePhase::Started;
|
||||||
|
if (Hand->CandidateFrames >= CandidateThreshold)
|
||||||
|
{
|
||||||
|
Hand->Phase = EHandGesturePhase::Started;
|
||||||
|
|
||||||
|
UE_LOG(LogTemp, Warning,
|
||||||
|
TEXT("[%s] CANDIDATE -> STARTED (%s)"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*BestGesture.ToString());
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case EGesturePhase::Ongoing:
|
}
|
||||||
|
UE_LOG(LogTemp, Warning,
|
||||||
|
TEXT("[%s] Candidate switched %s -> %s"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*Hand->ActiveGestureName.ToString(),
|
||||||
|
*BestGesture.ToString());
|
||||||
|
|
||||||
|
Hand->ActiveGestureName = BestGesture;
|
||||||
|
Hand->CandidateFrames = 1;
|
||||||
|
Hand->FailFrames = 0;
|
||||||
|
break;
|
||||||
|
case EHandGesturePhase::Ongoing: // Checking if the gesture is still ongoinga
|
||||||
|
//if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
|
||||||
|
//else Hand->FailFrames = 0;
|
||||||
if (Hand->ActiveGestureName != BestGesture)
|
if (Hand->ActiveGestureName != BestGesture)
|
||||||
{
|
{
|
||||||
Hand->FailFrames++;
|
Hand->FailFrames++;
|
||||||
|
|
||||||
if (Hand->FailFrames >= FailThreshold)
|
UE_LOG(LogTemp, Warning,
|
||||||
{
|
TEXT("[%s] Ongoing fail %s (%d/%d)"),
|
||||||
Hand->Phase = EGesturePhase::Ended;
|
*Hand->GetName(),
|
||||||
}
|
*Hand->ActiveGestureName.ToString(),
|
||||||
|
Hand->FailFrames,
|
||||||
|
FailThreshold);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
Hand->FailFrames = 0;
|
if (Hand->FailFrames > 0)
|
||||||
|
{
|
||||||
|
UE_LOG(LogTemp, Warning,
|
||||||
|
TEXT("[%s] Ongoing recovered %s"),
|
||||||
|
*Hand->GetName(),
|
||||||
|
*Hand->ActiveGestureName.ToString());
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
//case EHandGesturePhase::Started:
|
||||||
|
// break;
|
||||||
|
default: // If the gesture would be stuck somewhere
|
||||||
|
if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
|
||||||
|
|
||||||
|
if (Hand->FailFrames >= FailThreshold) Hand->ResetGesture(); // This optimaly shouldnt be called here
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Checking if a gesture has ended
|
||||||
|
if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
|
||||||
if (Hand->FailFrames >= FailThreshold)
|
|
||||||
Hand->Phase = EGesturePhase::Ended;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UGestureManagerComponent::TriggerAction(AHandActor* Hand)
|
void UGestureManagerComponent::TriggerAction(AHandActor* Hand)
|
||||||
{
|
{
|
||||||
if (!Hand->bUsedThisTick) return;
|
|
||||||
|
|
||||||
if (Hand->IsPaired())
|
if (Hand->bUsedThisTick) return;
|
||||||
{
|
|
||||||
AHandActor* Other = Hand->PairedHand;
|
if (TryExtendGestureAction(Hand)) return;
|
||||||
|
|
||||||
for (const FGestureBinding& Binding : GestureBindings)
|
for (const FGestureBinding& Binding : GestureBindings)
|
||||||
{
|
{
|
||||||
const bool bLeftMatch = (Hand->Handedness == EHandedness::Left && Hand->ActiveGestureName == Binding.LeftGesture &&
|
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|
||||||
Other->ActiveGestureName == Binding.RightGesture);
|
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
||||||
|
|
||||||
const bool bRightMatch = (Hand->Handedness == EHandedness::Right && Hand->ActiveGestureName == Binding.RightGesture &&
|
if (!bMatches) continue;
|
||||||
Other->ActiveGestureName == Binding.LeftGesture);
|
|
||||||
|
|
||||||
if (!(bLeftMatch || bRightMatch))
|
AGestureAction* NewAction = SpawnGestureAction(Binding.ActionName, Hand);
|
||||||
continue;
|
if (!NewAction) return;
|
||||||
|
|
||||||
// both must be ready
|
NewAction->InitAction(Hand, Binding, UEUnitsPerUnit, TableHeight);
|
||||||
if (Other->Phase != EGesturePhase::Started)
|
Hand->Phase = EHandGesturePhase::Ongoing;
|
||||||
continue;
|
|
||||||
|
|
||||||
// mark used
|
|
||||||
Hand->bUsedThisTick = true;
|
|
||||||
Other->bUsedThisTick = true;
|
|
||||||
|
|
||||||
Hand->Phase = EGesturePhase::Ongoing;
|
|
||||||
Other->Phase = EGesturePhase::Ongoing;
|
|
||||||
|
|
||||||
// spawn action (centralized factory recommended)
|
|
||||||
SpawnGestureAction(Binding.ActionName, Hand, Other);
|
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const FGestureBinding& Binding : GestureBindings)
|
|
||||||
|
void UGestureManagerComponent::UpdateOngoingActions()
|
||||||
{
|
{
|
||||||
const bool bMatch =
|
|
||||||
(Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName) ||
|
|
||||||
(Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
|
||||||
|
|
||||||
if (!bMatch)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
// if paired hand is actively running something, allow extension check later
|
|
||||||
if (Hand->IsPaired() && Hand->PairedHand && Hand->PairedHand->Phase == EGesturePhase::Ongoing)
|
|
||||||
{
|
|
||||||
for (const FExtendedGestureBinding& Ext : Binding.ExentdedGestures)
|
|
||||||
{
|
|
||||||
if (Ext.GestureName != Hand->ActiveGestureName)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (Ext.GestureAction == Binding.ActionName)
|
|
||||||
{
|
|
||||||
// extend current action
|
|
||||||
ExtendGestureAction(Binding.ActionName, Hand);
|
|
||||||
Hand->bUsedThisTick = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// branch: spawn new action linked to existing one
|
|
||||||
SpawnGestureActionWithParent(Ext.GestureAction, Hand, Hand->PairedHand);
|
|
||||||
Hand->bUsedThisTick = true;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// normal one-hand action
|
|
||||||
Hand->bUsedThisTick = true;
|
|
||||||
SpawnGestureAction(Binding.ActionName, Hand, nullptr);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void UGestureManagerComponent::UpdateOngoingAction()
|
|
||||||
{
|
|
||||||
for (AGestureAction* Action : ActiveGestureActions)
|
for (AGestureAction* Action : ActiveGestureActions)
|
||||||
{
|
{
|
||||||
if (!Action) continue;
|
if (!Action) continue;
|
||||||
@@ -338,18 +353,98 @@ void UGestureManagerComponent::UpdateOngoingAction()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void UGestureManagerComponent::SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B)
|
AGestureAction* UGestureManagerComponent::SpawnGestureAction(FName ActionName, AHandActor* Hand)
|
||||||
{
|
{
|
||||||
|
TSubclassOf<AGestureAction>* ActionClassPtr = ActionClassMap.Find(ActionName);
|
||||||
|
|
||||||
|
if (!ActionClassPtr || !Hand) return nullptr;
|
||||||
|
|
||||||
|
FActorSpawnParameters SpawnParams;
|
||||||
|
SpawnParams.Owner = GetOwner();
|
||||||
|
SpawnParams.SpawnCollisionHandlingOverride =
|
||||||
|
ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
|
||||||
|
|
||||||
|
AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
|
||||||
|
*ActionClassPtr,
|
||||||
|
FVector::ZeroVector,
|
||||||
|
FRotator::ZeroRotator,
|
||||||
|
SpawnParams);
|
||||||
|
|
||||||
|
if (!NewAction) return nullptr;
|
||||||
|
|
||||||
|
ActiveGestureActions.Add(NewAction);
|
||||||
|
|
||||||
|
Hand->ActiveAction = NewAction;
|
||||||
|
Hand->bUsedThisTick = true;
|
||||||
|
Hand->Phase = EHandGesturePhase::Ongoing;
|
||||||
|
|
||||||
|
return NewAction;
|
||||||
}
|
}
|
||||||
|
|
||||||
void UGestureManagerComponent::ExtendGestureAction(FString ActionName, AHandActor* Hand)
|
bool UGestureManagerComponent::TryExtendGestureAction(AHandActor* Hand)
|
||||||
{
|
{
|
||||||
|
if (!Hand->IsPaired() || Hand->PairedHand->Phase != EHandGesturePhase::Ongoing || !Hand->PairedHand->ActiveAction) return false;
|
||||||
|
|
||||||
|
if (TryJoinExistingAction(Hand)) return true;
|
||||||
|
if (TryCreateExtension(Hand)) return true;
|
||||||
|
|
||||||
|
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void UGestureManagerComponent::SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand)
|
bool UGestureManagerComponent::TryJoinExistingAction(AHandActor* Hand)
|
||||||
{
|
{
|
||||||
|
AHandActor* OtherHand = Hand->PairedHand;
|
||||||
|
AGestureAction* OtherAction = OtherHand->ActiveAction;
|
||||||
|
|
||||||
|
if (!OtherAction->bRequiresSecondHand) return false; // Check if its a Two Handed Action or an extension
|
||||||
|
|
||||||
|
FGestureBinding& Binding = OtherAction->Binding;
|
||||||
|
|
||||||
|
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|
||||||
|
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
||||||
|
|
||||||
|
if (!bMatches) return false;
|
||||||
|
|
||||||
|
OtherAction->AddOtherHand(Hand);
|
||||||
|
OtherHand->bUsedThisTick = true;
|
||||||
|
Hand->bUsedThisTick = true;
|
||||||
|
Hand->Phase = EHandGesturePhase::Ongoing;
|
||||||
|
Hand->ActiveAction = OtherAction;
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool UGestureManagerComponent::TryCreateExtension(AHandActor* Hand)
|
||||||
|
{
|
||||||
|
AHandActor* OtherHand = Hand->PairedHand;
|
||||||
|
AGestureAction* OtherAction = OtherHand->ActiveAction;
|
||||||
|
|
||||||
|
if (!OtherHand->ActiveAction->bExtendable) return false;
|
||||||
|
|
||||||
|
FName ActionName;
|
||||||
|
for (const FExtendedGestureBinding& Gesture : OtherAction->Binding.ExtendedGestures)
|
||||||
|
{
|
||||||
|
if (Gesture.GestureName == Hand->ActiveGestureName)
|
||||||
|
{
|
||||||
|
ActionName = Gesture.GestureAction;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ActionName.IsNone()) return false;
|
||||||
|
|
||||||
|
AGestureAction* NewAction = SpawnGestureAction(ActionName, Hand);
|
||||||
|
|
||||||
|
if (!NewAction) return false;
|
||||||
|
|
||||||
|
ActiveGestureActions.Add(NewAction);
|
||||||
|
|
||||||
|
NewAction->InitAction(Hand, OtherHand->ActiveAction->Binding, UEUnitsPerUnit, TableHeight);
|
||||||
|
OtherHand->bUsedThisTick = true;
|
||||||
|
Hand->bUsedThisTick = true;
|
||||||
|
OtherAction->AddExtension(NewAction);
|
||||||
|
NewAction->AddExtension(OtherHand->ActiveAction);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
#include "HandManager.h"
|
#include "HandManager.h"
|
||||||
#include "FingerTip.h"
|
#include "FingerTip.h"
|
||||||
#include "DrawDebugHelpers.h"
|
#include "DrawDebugHelpers.h"
|
||||||
|
#include "GestureAction.h"
|
||||||
|
|
||||||
|
|
||||||
static const int32 HandBones[][2] = {
|
static const int32 HandBones[][2] = {
|
||||||
@@ -31,14 +32,21 @@ AHandActor::AHandActor()
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void AHandActor::ResetGesture()
|
||||||
|
{
|
||||||
|
Phase = EHandGesturePhase::None;
|
||||||
|
ActiveGestureName = "";
|
||||||
|
CandidateFrames = 0;
|
||||||
|
FailFrames = 0;
|
||||||
|
if (IsValid(ActiveAction)) ActiveAction->EndGesture();
|
||||||
|
ActiveAction = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
// Called when the game starts or when spawned
|
// Called when the game starts or when spawned
|
||||||
void AHandActor::BeginPlay()
|
void AHandActor::BeginPlay()
|
||||||
{
|
{
|
||||||
Super::BeginPlay();
|
Super::BeginPlay();
|
||||||
GetWorld()->GetTimerManager().SetTimer(DeathTimer, this, &AHandActor::HandDeath, DeathTime, true);
|
GetWorld()->GetTimerManager().SetTimer(DeathTimer, this, &AHandActor::HandDeath, DeathTime, true);
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called every frame
|
// Called every frame
|
||||||
@@ -123,8 +131,6 @@ void AHandActor::UpdateHand(const FHandPacketHand& NewHand)
|
|||||||
Confidence = NewHand.Confidence;
|
Confidence = NewHand.Confidence;
|
||||||
HandId = NewHand.HandId;
|
HandId = NewHand.HandId;
|
||||||
|
|
||||||
//UE_LOG(LogTemp, Error, TEXT("Update Hand"));
|
|
||||||
|
|
||||||
for (int32 n = 0; n < 5; n++)
|
for (int32 n = 0; n < 5; n++)
|
||||||
{
|
{
|
||||||
FVector f = Points[FingerTipBones[n]];
|
FVector f = Points[FingerTipBones[n]];
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ AHandManager::AHandManager()
|
|||||||
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||||
PrimaryActorTick.bCanEverTick = true;
|
PrimaryActorTick.bCanEverTick = true;
|
||||||
GestureManager = CreateDefaultSubobject<UGestureManagerComponent>(TEXT("GestureManager"));
|
GestureManager = CreateDefaultSubobject<UGestureManagerComponent>(TEXT("GestureManager"));
|
||||||
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called when the game starts or when spawned
|
// Called when the game starts or when spawned
|
||||||
@@ -18,6 +18,8 @@ void AHandManager::BeginPlay()
|
|||||||
{
|
{
|
||||||
Super::BeginPlay();
|
Super::BeginPlay();
|
||||||
|
|
||||||
|
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
|
||||||
|
|
||||||
if (!IsValid(HandActorClass))
|
if (!IsValid(HandActorClass))
|
||||||
{
|
{
|
||||||
HandActorClass = AHandActor::StaticClass();
|
HandActorClass = AHandActor::StaticClass();
|
||||||
@@ -69,7 +71,7 @@ void AHandManager::AddDeadHand(AHandActor* DeadHand)
|
|||||||
DeadHands.Add(DeadHand);
|
DeadHands.Add(DeadHand);
|
||||||
}
|
}
|
||||||
|
|
||||||
TArray<AHandActor*> AHandManager::GetHandList()
|
TArray<AHandActor*>& AHandManager::GetHandList()
|
||||||
{
|
{
|
||||||
return HandList;
|
return HandList;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
|
||||||
|
#include "MyGestureActions.h"
|
||||||
|
#include "HandActor.h"
|
||||||
|
#include "TableIconActor.h"
|
||||||
|
#include "MyTableIconActors.h"
|
||||||
|
|
||||||
|
|
||||||
|
void AGestureAction_Ruler::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit, float TableHeight)
|
||||||
|
{
|
||||||
|
Super::InitAction_Implementation(Hand, NewBinding, NewUEUnitsPerUnit, TableHeight);
|
||||||
|
DesiredHeight = -500;
|
||||||
|
StartPoint = HandA->Points[8];
|
||||||
|
StartPoint.Z = DesiredHeight;
|
||||||
|
|
||||||
|
FActorSpawnParameters SpawnParams;
|
||||||
|
SpawnParams.Owner = this;
|
||||||
|
SpawnParams.SpawnCollisionHandlingOverride = ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
|
||||||
|
|
||||||
|
if (LineIconClass)
|
||||||
|
{
|
||||||
|
|
||||||
|
LineIcon = GetWorld()->SpawnActor<ATableIconLineActor>(LineIconClass, StartPoint, FRotator::ZeroRotator, SpawnParams);
|
||||||
|
|
||||||
|
if (LineIcon)
|
||||||
|
{
|
||||||
|
LineIcon->OwningAction = this;
|
||||||
|
LineIcon->bPersistOnActionEnd = true;
|
||||||
|
LineIcon->LineThickness = 50;
|
||||||
|
Icons.Add(LineIcon);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RulerPoints.Add(FVector::ZeroVector);
|
||||||
|
RulerPoints.Add(FVector::ZeroVector);
|
||||||
|
|
||||||
|
if (LabelIconClass)
|
||||||
|
{
|
||||||
|
LabelIcon = GetWorld()->SpawnActor<ATableIconLabelActor>(LabelIconClass, StartPoint, FRotator::ZeroRotator, SpawnParams);
|
||||||
|
|
||||||
|
if (LabelIcon)
|
||||||
|
{
|
||||||
|
LabelIcon->OwningAction = this;
|
||||||
|
LabelIcon->bPersistOnActionEnd = true;
|
||||||
|
LabelIcon->UpdateLabel(StartPoint, 0.f);
|
||||||
|
Icons.Add(LabelIcon);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void AGestureAction_Ruler::UpdateAction_Implementation()
|
||||||
|
{
|
||||||
|
if (!HandA || !IsValid(LineIcon) || !IsValid(LabelIcon)) return;
|
||||||
|
const FVector Current = HandA->Points[8];
|
||||||
|
|
||||||
|
//if (FVector::DistSquared(Current, RulerPoints[RulerPoints.Num() - 1]) < FMath::Square(UEUnitsPerUnit)) return;
|
||||||
|
RulerPoints.Last() = Current - StartPoint;
|
||||||
|
RulerPoints.Last().Z = 0;
|
||||||
|
LineIcon->SetPoints(RulerPoints);
|
||||||
|
|
||||||
|
// RulerPoints are local to the line actor, so convert back to world
|
||||||
|
FVector LabelPosition = StartPoint + (RulerPoints.Last() * 0.5f);
|
||||||
|
LabelPosition.Z = 10;
|
||||||
|
LabelIcon->UpdateLabel(LabelPosition, GetSnappedUnits(GetTotalLength()));
|
||||||
|
}
|
||||||
|
|
||||||
|
float AGestureAction_Ruler::GetSnappedUnits(float RawDistance) const
|
||||||
|
{
|
||||||
|
if (UEUnitsPerUnit <= 0.f) return 0.f;
|
||||||
|
return FMath::RoundToFloat(RawDistance / UEUnitsPerUnit);
|
||||||
|
}
|
||||||
|
|
||||||
|
float AGestureAction_Ruler::GetTotalLength() const
|
||||||
|
{
|
||||||
|
float Total = 0.f;
|
||||||
|
for (int32 i = 0; i < RulerPoints.Num() - 1; i++)
|
||||||
|
Total += FVector::Dist(RulerPoints[i], RulerPoints[i + 1]);
|
||||||
|
return Total;
|
||||||
|
}
|
||||||
|
|
||||||
|
void AGestureAction_Ruler::AddRulerPoint()
|
||||||
|
{
|
||||||
|
FVector Current = HandA->Points[8];
|
||||||
|
RulerPoints.Add(Current - StartPoint);
|
||||||
|
}
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
|
||||||
|
#include "MyTableIconActors.h"
|
||||||
|
#include "KismetProceduralMeshLibrary.h"
|
||||||
|
#include "TableIconActor.h"
|
||||||
|
|
||||||
|
#include "Kismet/GameplayStatics.h"
|
||||||
|
#include "Camera/CameraActor.h"
|
||||||
|
|
||||||
|
ATableIconLineActor::ATableIconLineActor()
|
||||||
|
{
|
||||||
|
PrimaryActorTick.bCanEverTick = false; // purely driven by SetPoints
|
||||||
|
|
||||||
|
USceneComponent* Root = CreateDefaultSubobject<USceneComponent>(TEXT("Root"));
|
||||||
|
SetRootComponent(Root);
|
||||||
|
|
||||||
|
MeshComponent = CreateDefaultSubobject<UProceduralMeshComponent>(TEXT("LineMesh"));
|
||||||
|
MeshComponent->SetupAttachment(RootComponent);
|
||||||
|
MeshComponent->SetCastShadow(false);
|
||||||
|
MeshComponent->SetCollisionEnabled(ECollisionEnabled::NoCollision);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ATableIconLineActor::SetPoints(const TArray<FVector>& NewPoints)
|
||||||
|
{
|
||||||
|
const FTransform& ActorTransform = GetActorTransform();
|
||||||
|
|
||||||
|
Points.Reset(NewPoints.Num());
|
||||||
|
|
||||||
|
Points = NewPoints;
|
||||||
|
|
||||||
|
MeshComponent->ClearMeshSection(0);
|
||||||
|
|
||||||
|
BuildLineMesh();
|
||||||
|
|
||||||
|
if (LineMaterial) MeshComponent->SetMaterial(0, LineMaterial);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ATableIconLineActor::BuildLineMesh()
|
||||||
|
{
|
||||||
|
TArray<FVector> Vertices;
|
||||||
|
TArray<int32> Triangles;
|
||||||
|
TArray<FVector> Normals;
|
||||||
|
TArray<FVector2D> UVs;
|
||||||
|
TArray<FColor> Colors;
|
||||||
|
TArray<FProcMeshTangent> Tangents;
|
||||||
|
|
||||||
|
FVector Direction = FVector::ZeroVector;
|
||||||
|
const float HalfLineThickness = LineThickness * 0.5f;
|
||||||
|
|
||||||
|
for (int32 i = 0; i < Points.Num() - 1; i++) {
|
||||||
|
const FVector& PointA = Points[i];
|
||||||
|
const FVector& PointB = Points[i + 1];
|
||||||
|
|
||||||
|
Direction = (PointB - PointA).GetSafeNormal();
|
||||||
|
|
||||||
|
const FVector Right = FVector::CrossProduct(FVector::UpVector, Direction).GetSafeNormal();
|
||||||
|
|
||||||
|
const FVector Offset = Right * (LineThickness * 0.5f);
|
||||||
|
|
||||||
|
int32 Base = Vertices.Num();
|
||||||
|
|
||||||
|
Vertices.Add(PointA - Offset);
|
||||||
|
Vertices.Add(PointA + Offset);
|
||||||
|
Vertices.Add(PointB + Offset);
|
||||||
|
Vertices.Add(PointB - Offset);
|
||||||
|
|
||||||
|
Triangles.Add(Base + 0);
|
||||||
|
Triangles.Add(Base + 1);
|
||||||
|
Triangles.Add(Base + 2);
|
||||||
|
|
||||||
|
Triangles.Add(Base + 0);
|
||||||
|
Triangles.Add(Base + 2);
|
||||||
|
Triangles.Add(Base + 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
for (int i = 0; i < Vertices.Num(); i++)
|
||||||
|
{
|
||||||
|
Normals.Add(FVector::UpVector);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < Vertices.Num(); i++)
|
||||||
|
{
|
||||||
|
UVs.Add(FVector2D::ZeroVector);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < Vertices.Num(); i++)
|
||||||
|
{
|
||||||
|
Colors.Add(FColor::Red);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < Vertices.Num(); i++)
|
||||||
|
{
|
||||||
|
Tangents.Add(FProcMeshTangent(Direction, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
MeshComponent->CreateMeshSection(0, Vertices, Triangles, Normals, UVs, Colors, Tangents, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
ATableIconWidgetActor::ATableIconWidgetActor()
|
||||||
|
{
|
||||||
|
USceneComponent* Root = CreateDefaultSubobject<USceneComponent>(TEXT("Root"));
|
||||||
|
SetRootComponent(Root);
|
||||||
|
|
||||||
|
PrimaryActorTick.bCanEverTick = false;
|
||||||
|
|
||||||
|
WidgetComponent = CreateDefaultSubobject<UWidgetComponent>(TEXT("WidgetComponent"));
|
||||||
|
WidgetComponent->SetupAttachment(RootComponent);
|
||||||
|
WidgetComponent->SetDrawAtDesiredSize(true);
|
||||||
|
WidgetComponent->SetWidgetSpace(EWidgetSpace::World);
|
||||||
|
WidgetComponent->SetCollisionEnabled(ECollisionEnabled::NoCollision);
|
||||||
|
|
||||||
|
WidgetInteraction = CreateDefaultSubobject<UWidgetInteractionComponent>(TEXT("WidgetInteraction"));
|
||||||
|
WidgetInteraction->SetupAttachment(RootComponent);
|
||||||
|
WidgetInteraction->InteractionSource = EWidgetInteractionSource::Custom;
|
||||||
|
WidgetInteraction->InteractionDistance = 10.f;
|
||||||
|
|
||||||
|
SetWidgetClass(LabelWidgetClass);
|
||||||
|
}
|
||||||
|
|
||||||
|
void ATableIconWidgetActor::SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass)
|
||||||
|
{
|
||||||
|
if (!InWidgetClass) return;
|
||||||
|
WidgetComponent->SetWidgetClass(InWidgetClass);
|
||||||
|
WidgetComponent->InitWidget();
|
||||||
|
}
|
||||||
|
|
||||||
|
UUserWidget* ATableIconWidgetActor::GetWidget() const
|
||||||
|
{
|
||||||
|
return WidgetComponent->GetUserWidgetObject();
|
||||||
|
}
|
||||||
|
|
||||||
|
//--------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
void ATableIconLabelActor::UpdateLabel(const FVector& WorldLocation, float Value)
|
||||||
|
{
|
||||||
|
SetActorLocation(WorldLocation);
|
||||||
|
SetLabelText(FString::Printf(TEXT("%.1f%s"), Value, *UnitSuffix));
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
|
||||||
|
#include "TableIconActor.h"
|
||||||
|
|
||||||
|
// Sets default values
|
||||||
|
ATableIconActor::ATableIconActor()
|
||||||
|
{
|
||||||
|
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||||
|
PrimaryActorTick.bCanEverTick = false;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called when the game starts or when spawned
|
||||||
|
void ATableIconActor::BeginPlay()
|
||||||
|
{
|
||||||
|
Super::BeginPlay();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
void ATableIconActor::OnActionEnded_Implementation()
|
||||||
|
{
|
||||||
|
if (!bPersistOnActionEnd) Destroy();
|
||||||
|
}
|
||||||
|
|
||||||
|
void ATableIconActor::OnActionUpdated_Implementation()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -4,9 +4,12 @@
|
|||||||
|
|
||||||
#include "CoreMinimal.h"
|
#include "CoreMinimal.h"
|
||||||
#include "GameFramework/Actor.h"
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "GestureData.h"
|
||||||
#include "GestureAction.generated.h"
|
#include "GestureAction.generated.h"
|
||||||
|
|
||||||
class AHandActor;
|
class AHandActor;
|
||||||
|
class ATableIconActor;
|
||||||
|
class AHandManager;
|
||||||
|
|
||||||
UCLASS()
|
UCLASS()
|
||||||
class TABLE_API AGestureAction : public AActor
|
class TABLE_API AGestureAction : public AActor
|
||||||
@@ -22,27 +25,23 @@ protected:
|
|||||||
virtual void BeginPlay() override;
|
virtual void BeginPlay() override;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
// Called every frame
|
|
||||||
virtual void Tick(float DeltaTime) override;
|
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite) float UEUnitsPerUnit = 10.f;
|
||||||
TMap<FName, FVector> Anchors;
|
UPROPERTY(EditDefaultsOnly) bool bRequiresSecondHand = false;
|
||||||
|
UPROPERTY(EditDefaultsOnly) bool bExtendable = false;
|
||||||
|
UPROPERTY(EditDefaultsOnly) FGestureBinding Binding;
|
||||||
|
UPROPERTY(BlueprintReadWrite) TArray<ATableIconActor*> Icons;
|
||||||
|
UPROPERTY(BlueprintReadWrite) AHandActor* HandA;
|
||||||
|
UPROPERTY(BlueprintReadWrite) AHandActor* HandB;
|
||||||
|
UPROPERTY(BlueprintReadWrite) AGestureAction* LinkedActions;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Stuff") AHandManager* HandManager;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void InitAction(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit, float TableHeight);
|
||||||
AHandActor* HandA;
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void UpdateAction();
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void AbortAction();
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void AddExtension(AGestureAction* OtherAction);
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void AddOtherHand(AHandActor* OtherHand);
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintNativeEvent) void EndGesture();
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
void DestroyIcons();
|
||||||
AHandActor* HandB;
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
void UpdateAction();
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
void AbortAction();
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
void AddExtension(AHandActor* OtherHand);
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
void EndGesture();
|
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ enum class EHandedness : uint8
|
|||||||
};
|
};
|
||||||
|
|
||||||
UENUM(BlueprintType)
|
UENUM(BlueprintType)
|
||||||
enum class EGesturePhase : uint8
|
enum class EHandGesturePhase : uint8
|
||||||
{
|
{
|
||||||
None,
|
None,
|
||||||
Candidate,
|
Candidate,
|
||||||
@@ -54,10 +54,10 @@ struct FExtendedGestureBinding
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
FString GestureName;
|
FName GestureName;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
FString GestureAction;
|
FName GestureAction;
|
||||||
};
|
};
|
||||||
|
|
||||||
// What gets parsed from the JSON
|
// What gets parsed from the JSON
|
||||||
@@ -67,15 +67,15 @@ struct FGestureBinding
|
|||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
FString ActionName;
|
FName ActionName;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
FString LeftGesture;
|
FName LeftGesture;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
FString RightGesture;
|
FName RightGesture;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(BlueprintReadOnly)
|
||||||
TArray<FExtendedGestureBinding> ExentdedGestures;
|
TArray<FExtendedGestureBinding> ExtendedGestures;
|
||||||
|
|
||||||
};
|
};
|
||||||
@@ -31,8 +31,7 @@ class TABLE_API UGestureDefinition : public UObject
|
|||||||
public:
|
public:
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UPROPERTY(BlueprintReadWrite)
|
||||||
FString GestureName;
|
FName GestureName;
|
||||||
|
|
||||||
|
|
||||||
// This checks the gesture and returns an aproximate scor of how well the Gesture was exectuded, needs to be overwritten in child classes.
|
// This checks the gesture and returns an aproximate scor of how well the Gesture was exectuded, needs to be overwritten in child classes.
|
||||||
UFUNCTION(BlueprintCallable)
|
UFUNCTION(BlueprintCallable)
|
||||||
@@ -50,13 +49,96 @@ protected:
|
|||||||
float GetFingersAverage(const AHandActor* Hand, const TArray<EHandFinger>& Fingers, bool bExtended) const;
|
float GetFingersAverage(const AHandActor* Hand, const TArray<EHandFinger>& Fingers, bool bExtended) const;
|
||||||
FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const;
|
FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const;
|
||||||
const FVector& GetPoint(const AHandActor* Hand, int32 Index) const;
|
const FVector& GetPoint(const AHandActor* Hand, int32 Index) const;
|
||||||
|
bool IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold = 0.7f) const;
|
||||||
};
|
};
|
||||||
|
|
||||||
UCLASS()
|
UCLASS()
|
||||||
class TABLE_API UGesture_PinchPoint : public UGestureDefinition
|
class TABLE_API UGesture_PinchIndex : public UGestureDefinition
|
||||||
{
|
{
|
||||||
GENERATED_BODY()
|
GENERATED_BODY()
|
||||||
|
|
||||||
public:
|
public:
|
||||||
virtual float Evaluate(AHandActor* Hand) const override;
|
virtual float Evaluate(AHandActor* Hand) const override;
|
||||||
|
virtual void PostInitProperties() override
|
||||||
|
{
|
||||||
|
Super::PostInitProperties();
|
||||||
|
|
||||||
|
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||||
|
{
|
||||||
|
GestureName = TEXT("PinchIndex");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API UGesture_PinchMiddleIndex : public UGestureDefinition
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
virtual float Evaluate(AHandActor* Hand) const override;
|
||||||
|
virtual void PostInitProperties() override
|
||||||
|
{
|
||||||
|
Super::PostInitProperties();
|
||||||
|
|
||||||
|
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||||
|
{
|
||||||
|
GestureName = FName("PinchMiddle");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API UGesture_PointIndex : public UGestureDefinition
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
virtual float Evaluate(AHandActor* Hand) const override;
|
||||||
|
virtual void PostInitProperties() override
|
||||||
|
{
|
||||||
|
Super::PostInitProperties();
|
||||||
|
|
||||||
|
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||||
|
{
|
||||||
|
GestureName = FName("PointIndex");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API UGesture_PointMiddle : public UGestureDefinition
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
virtual float Evaluate(AHandActor* Hand) const override;
|
||||||
|
virtual void PostInitProperties() override
|
||||||
|
{
|
||||||
|
Super::PostInitProperties();
|
||||||
|
|
||||||
|
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||||
|
{
|
||||||
|
GestureName = TEXT("PointMiddle");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API UGesture_PointIndexMiddle : public UGestureDefinition
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
virtual float Evaluate(AHandActor* Hand) const override;
|
||||||
|
virtual void PostInitProperties() override
|
||||||
|
{
|
||||||
|
Super::PostInitProperties();
|
||||||
|
|
||||||
|
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||||
|
{
|
||||||
|
GestureName = TEXT("PointIndexMiddle");
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
@@ -7,7 +7,7 @@
|
|||||||
#include "GestureData.h"
|
#include "GestureData.h"
|
||||||
#include "GestureManagerComponent.generated.h"
|
#include "GestureManagerComponent.generated.h"
|
||||||
|
|
||||||
|
class AHandManager;
|
||||||
|
|
||||||
UCLASS(ClassGroup = "Gesture", BlueprintType, Blueprintable, ClassGroup=(Custom), meta=(BlueprintSpawnableComponent) )
|
UCLASS(ClassGroup = "Gesture", BlueprintType, Blueprintable, ClassGroup=(Custom), meta=(BlueprintSpawnableComponent) )
|
||||||
class TABLE_API UGestureManagerComponent : public UActorComponent
|
class TABLE_API UGestureManagerComponent : public UActorComponent
|
||||||
@@ -24,31 +24,25 @@ protected:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly)
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale") float UEUnitsPerUnit = 10.f; // set via calibration rectangle
|
||||||
class AHandManager* HandManager;
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale") float TableHeight = -650.f;
|
||||||
|
UPROPERTY(BlueprintReadOnly) AHandManager* HandManager;
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable, Category = "Gesture")
|
UFUNCTION(BlueprintCallable, Category = "Gesture") void EvaluateHands();
|
||||||
void EvaluateHands();
|
|
||||||
|
|
||||||
UFUNCTION()
|
UFUNCTION() void OnHandAdded(AHandActor* Hand);
|
||||||
void OnHandAdded(AHandActor* Hand);
|
|
||||||
|
|
||||||
UFUNCTION()
|
UFUNCTION() void OnHandRemoved(AHandActor* Hand);
|
||||||
void OnHandRemoved(AHandActor* Hand);
|
|
||||||
|
|
||||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
UPROPERTY(EditDefaultsOnly, Category = "Gesture") FString BindingConfigPath; // path to the per-game JSON
|
||||||
FString BindingConfigPath; // path to the per-game JSON
|
|
||||||
|
|
||||||
// Loaded from JSON — gesture name -> definition object
|
// Loaded from JSON — gesture name -> definition object
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
UPROPERTY(BlueprintReadOnly, Category = "Gesture") TArray<UGestureDefinition*> GestureDefinitions;
|
||||||
TArray<UGestureDefinition*> GestureDefinitions;
|
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
UPROPERTY(BlueprintReadOnly, Category = "Gesture") TArray<AGestureAction*> ActiveGestureActions;
|
||||||
TArray<AGestureAction*> ActiveGestureActions;
|
|
||||||
|
|
||||||
// Populated on BeginPlay from JSON
|
// Populated on BeginPlay from JSON
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
UPROPERTY(BlueprintReadOnly, Category = "Gesture") TArray<FGestureBinding> GestureBindings;
|
||||||
TArray<FGestureBinding> GestureBindings;
|
|
||||||
void LoadBindings(const FString& FilePath);
|
void LoadBindings(const FString& FilePath);
|
||||||
|
|
||||||
void Initialize(UDataTable* InGestureTable, UDataTable* InActionTable);
|
void Initialize(UDataTable* InGestureTable, UDataTable* InActionTable);
|
||||||
@@ -56,29 +50,26 @@ public:
|
|||||||
void BuildActionMap();
|
void BuildActionMap();
|
||||||
|
|
||||||
// Populated on BeginPlay from the tables
|
// Populated on BeginPlay from the tables
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
UPROPERTY(BlueprintReadOnly, Category = "Gesture") TMap<FName, UGestureDefinition*> DefinitionMap;
|
||||||
TMap<FString, UGestureDefinition*> DefinitionMap;
|
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
UPROPERTY(BlueprintReadOnly, Category = "Gesture") TMap<FName, TSubclassOf<AGestureAction>> ActionClassMap;
|
||||||
TMap<FString, TSubclassOf<AGestureAction>> ActionClassMap;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
// GestureManager.h — add these
|
//
|
||||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
UPROPERTY(EditDefaultsOnly, Category = "Gesture") UDataTable* GestureDefinitionTable;
|
||||||
UDataTable* GestureDefinitionTable;
|
|
||||||
|
|
||||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
UPROPERTY(EditDefaultsOnly, Category = "Gesture") UDataTable* GestureActionTable;
|
||||||
UDataTable* GestureActionTable;
|
|
||||||
|
|
||||||
|
|
||||||
void CheckGestureDefinitions(TArray<AHandActor*>& Hands);
|
void CheckGestureDefinitions(TArray<AHandActor*>& Hands);
|
||||||
void TriggerAction(AHandActor* Hand);
|
void TriggerAction(AHandActor* Hand);
|
||||||
void UpdateOngoingAction();
|
void UpdateOngoingActions();
|
||||||
void EndGesture();
|
|
||||||
|
|
||||||
void SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B);
|
AGestureAction* SpawnGestureAction(FName ActionName, AHandActor* Hand);
|
||||||
void ExtendGestureAction(FString ActionName, AHandActor* Hand);
|
bool TryExtendGestureAction(AHandActor* Hand);
|
||||||
void SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand);
|
|
||||||
|
bool TryJoinExistingAction(AHandActor* Hand);
|
||||||
|
bool TryCreateExtension(AHandActor* Hand);
|
||||||
|
|
||||||
static constexpr int32 CandidateThreshold = 8;
|
static constexpr int32 CandidateThreshold = 8;
|
||||||
static constexpr int32 FailThreshold = 5;
|
static constexpr int32 FailThreshold = 5;
|
||||||
|
|||||||
@@ -11,6 +11,7 @@
|
|||||||
|
|
||||||
class AHandManager;
|
class AHandManager;
|
||||||
class AFingerTip;
|
class AFingerTip;
|
||||||
|
class AGestureAction;
|
||||||
|
|
||||||
UCLASS()
|
UCLASS()
|
||||||
class TABLE_API AHandActor : public AActor
|
class TABLE_API AHandActor : public AActor
|
||||||
@@ -47,10 +48,13 @@ public:
|
|||||||
// Gestures
|
// Gestures
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UPROPERTY(BlueprintReadWrite)
|
||||||
EGesturePhase Phase = EGesturePhase::None;
|
EHandGesturePhase Phase = EHandGesturePhase::None;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UPROPERTY(BlueprintReadWrite)
|
||||||
FString ActiveGestureName;
|
FName ActiveGestureName;
|
||||||
|
|
||||||
|
UPROPERTY(BlueprintReadWrite)
|
||||||
|
AGestureAction* ActiveAction;
|
||||||
|
|
||||||
UPROPERTY(BlueprintReadWrite)
|
UPROPERTY(BlueprintReadWrite)
|
||||||
TMap<FString, FVector> Anchors;
|
TMap<FString, FVector> Anchors;
|
||||||
@@ -67,15 +71,7 @@ public:
|
|||||||
bool bUsedThisTick = false;
|
bool bUsedThisTick = false;
|
||||||
|
|
||||||
bool IsPaired() const { return PairedHand != nullptr; }
|
bool IsPaired() const { return PairedHand != nullptr; }
|
||||||
|
void ResetGesture();
|
||||||
void ResetGesture()
|
|
||||||
{
|
|
||||||
Phase = EGesturePhase::None;
|
|
||||||
ActiveGestureName = "";
|
|
||||||
CandidateFrames = 0;
|
|
||||||
FailFrames = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
// Called when the game starts or when spawned
|
// Called when the game starts or when spawned
|
||||||
|
|||||||
@@ -34,49 +34,29 @@ public:
|
|||||||
|
|
||||||
void SpawnNewHand(const FVector& PalmCenter, const FHandPacketHand& NewHand);
|
void SpawnNewHand(const FVector& PalmCenter, const FHandPacketHand& NewHand);
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand Tracking|Debug")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") ADetectionManager* DM;
|
||||||
ADetectionManager* DM;
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") TArray<AHandActor*> HandList;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") TArray<AHandActor*> DeadHands;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") int32 HandAllocationDistance = FMath::Square(1000);;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") TSubclassOf<class AHandActor> HandActorClass;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand") TSubclassOf<class AFingerTip> FingerTipClass;
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand")
|
UPROPERTY(BlueprintAssignable) FOnHandAdded OnHandAdded;
|
||||||
TArray<AHandActor*> HandList;
|
UPROPERTY(BlueprintAssignable) FOnHandRemoved OnHandRemoved;
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand")
|
UFUNCTION(BlueprintCallable) void AddDeadHand(AHandActor* DeadHand);
|
||||||
TArray<AHandActor*> DeadHands;
|
UFUNCTION(BlueprintCallable, BlueprintPure) TArray<AHandActor*>& GetHandList();
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand")
|
|
||||||
int32 HandAllocationDistance = FMath::Square(1000);;
|
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand")
|
|
||||||
TSubclassOf<class AHandActor> HandActorClass;
|
|
||||||
|
|
||||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Hand")
|
|
||||||
TSubclassOf<class AFingerTip> FingerTipClass;
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
void AddDeadHand(AHandActor* DeadHand);
|
|
||||||
|
|
||||||
UPROPERTY(BlueprintAssignable)
|
|
||||||
FOnHandAdded OnHandAdded;
|
|
||||||
|
|
||||||
UPROPERTY(BlueprintAssignable)
|
|
||||||
FOnHandRemoved OnHandRemoved;
|
|
||||||
|
|
||||||
UFUNCTION(BlueprintCallable)
|
|
||||||
TArray<AHandActor*> GetHandList();
|
|
||||||
|
|
||||||
//Gestures
|
//Gestures
|
||||||
|
UPROPERTY(VisibleAnywhere, Category = "Gesture") UGestureManagerComponent* GestureManager;
|
||||||
UPROPERTY(VisibleAnywhere, Category = "Gesture")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture") UDataTable* GestureDefinitionTable;
|
||||||
UGestureManagerComponent* GestureManager;
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture") UDataTable* GestureActionTable;
|
||||||
|
|
||||||
const TArray<AHandActor*>& GetHandList() const { return HandList; }
|
const TArray<AHandActor*>& GetHandList() const { return HandList; }
|
||||||
|
|
||||||
// GestureManager.h — add these
|
//Gesture Actions, Icons
|
||||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale") float UEUnitsPerUnit = 10.f;
|
||||||
UDataTable* GestureDefinitionTable;
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale") float TableHeight = -650.f;
|
||||||
|
|
||||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
|
||||||
UDataTable* GestureActionTable;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void RemoveDeadHands();
|
void RemoveDeadHands();
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "GestureAction.h"
|
||||||
|
#include "MyGestureActions.generated.h"
|
||||||
|
|
||||||
|
class ATableIconLineActor;
|
||||||
|
class ATableIconLabelActor;
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API AMyGestureActions : public AGestureAction
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
//virtual void InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit) override;
|
||||||
|
//virtual void UpdateAction_Implementation() override;
|
||||||
|
//virtual void EndGesture_Implementation() override;
|
||||||
|
//virtual void AddOtherHand_Implementation(AHandActor* OtherHand) override;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
UCLASS(Blueprintable)
|
||||||
|
class TABLE_API AGestureAction_Ruler : public AGestureAction
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
// Set these in BP defaults — point at your BP subclasses
|
||||||
|
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite, Category = "Ruler") TSubclassOf<ATableIconLineActor> LineIconClass;
|
||||||
|
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite, Category = "Ruler") TSubclassOf<ATableIconLabelActor> LabelIconClass;
|
||||||
|
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite, Category = "Ruler") FString UnitSuffix = TEXT("\"");
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Ruler") TArray<FVector> RulerPoints;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Ruler") float DesiredHeight = -650.f;
|
||||||
|
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintPure) float GetTotalLength() const;
|
||||||
|
UFUNCTION(BlueprintCallable) void AddRulerPoint();
|
||||||
|
|
||||||
|
virtual void InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit, float TableHeight) override;
|
||||||
|
virtual void UpdateAction_Implementation() override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
UPROPERTY() ATableIconLineActor* LineIcon = nullptr;
|
||||||
|
UPROPERTY() ATableIconLabelActor* LabelIcon = nullptr;
|
||||||
|
UPROPERTY() FVector StartPoint;
|
||||||
|
|
||||||
|
float GetSnappedUnits(float RawDistance) const;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "TableIconActor.h"
|
||||||
|
#include "ProceduralMeshComponent.h"
|
||||||
|
#include "Components/WidgetComponent.h"
|
||||||
|
#include "Components/WidgetInteractionComponent.h"
|
||||||
|
#include "MyTableIconActors.generated.h"
|
||||||
|
|
||||||
|
class UWidgetComponent;
|
||||||
|
class UWidgetInteractionComponent;
|
||||||
|
|
||||||
|
|
||||||
|
UCLASS(Blueprintable)
|
||||||
|
class TABLE_API ATableIconLineActor : public ATableIconActor
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
ATableIconLineActor();
|
||||||
|
|
||||||
|
UFUNCTION(BlueprintCallable) void SetPoints(const TArray<FVector>& InPoints);
|
||||||
|
|
||||||
|
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Line") float LineThickness = 10.f;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Line") FLinearColor LineColor = FLinearColor::Blue;
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Line") UMaterialInterface* LineMaterial = nullptr;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
UPROPERTY(BlueprintReadOnly, Category = "Line") TArray<FVector> Points;
|
||||||
|
|
||||||
|
private:
|
||||||
|
UPROPERTY() UProceduralMeshComponent* MeshComponent;
|
||||||
|
|
||||||
|
void BuildLineMesh();
|
||||||
|
};
|
||||||
|
|
||||||
|
//--------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
UCLASS(Blueprintable)
|
||||||
|
class TABLE_API ATableIconWidgetActor : public ATableIconActor
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
ATableIconWidgetActor();
|
||||||
|
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly) UWidgetComponent* WidgetComponent;
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly) UWidgetInteractionComponent* WidgetInteraction;
|
||||||
|
UPROPERTY(VisibleAnywhere, BlueprintReadOnly) TSubclassOf<UUserWidget> LabelWidgetClass;
|
||||||
|
|
||||||
|
UFUNCTION(BlueprintCallable) void SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass);
|
||||||
|
UFUNCTION(BlueprintCallable, BlueprintPure) UUserWidget* GetWidget() const;
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
//--------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
UCLASS(Blueprintable)
|
||||||
|
class TABLE_API ATableIconLabelActor : public ATableIconWidgetActor
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Label") FString UnitSuffix = TEXT("\"");
|
||||||
|
|
||||||
|
UFUNCTION(BlueprintImplementableEvent, BlueprintCallable, Category = "Label") void SetLabelText(const FString& Text);
|
||||||
|
UFUNCTION(BlueprintCallable, Category = "Label") void UpdateLabel(const FVector& WorldLocation, float Value);
|
||||||
|
|
||||||
|
};
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
// Fill out your copyright notice in the Description page of Project Settings.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "CoreMinimal.h"
|
||||||
|
#include "GameFramework/Actor.h"
|
||||||
|
#include "TableIconActor.generated.h"
|
||||||
|
|
||||||
|
class AGestureAction;
|
||||||
|
|
||||||
|
UCLASS()
|
||||||
|
class TABLE_API ATableIconActor : public AActor
|
||||||
|
{
|
||||||
|
GENERATED_BODY()
|
||||||
|
|
||||||
|
public:
|
||||||
|
// Sets default values for this actor's properties
|
||||||
|
ATableIconActor();
|
||||||
|
|
||||||
|
protected:
|
||||||
|
// Called when the game starts or when spawned
|
||||||
|
virtual void BeginPlay() override;
|
||||||
|
|
||||||
|
public:
|
||||||
|
// If true, GestureAction will not destroy this on EndGesture
|
||||||
|
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite) bool bPersistOnActionEnd = false;
|
||||||
|
UPROPERTY(EditAnywhere) AGestureAction* OwningAction;
|
||||||
|
|
||||||
|
// Called by GestureAction on end — BP overrides this for fade/lock behavior
|
||||||
|
UFUNCTION(BlueprintNativeEvent) void OnActionEnded();
|
||||||
|
|
||||||
|
// Called by GestureAction every tick
|
||||||
|
UFUNCTION(BlueprintNativeEvent) void OnActionUpdated();
|
||||||
|
|
||||||
|
|
||||||
|
};
|
||||||
@@ -20,7 +20,8 @@ public class Table : ModuleRules
|
|||||||
"UMG" ,
|
"UMG" ,
|
||||||
"Json",
|
"Json",
|
||||||
"JsonUtilities",
|
"JsonUtilities",
|
||||||
"ImageWrapper"
|
"ImageWrapper",
|
||||||
|
"ProceduralMeshComponent"
|
||||||
});
|
});
|
||||||
|
|
||||||
PrivateDependencyModuleNames.AddRange(new string[] { });
|
PrivateDependencyModuleNames.AddRange(new string[] { });
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ public class TableEditorTarget : TargetRules
|
|||||||
public TableEditorTarget( TargetInfo Target) : base(Target)
|
public TableEditorTarget( TargetInfo Target) : base(Target)
|
||||||
{
|
{
|
||||||
Type = TargetType.Editor;
|
Type = TargetType.Editor;
|
||||||
DefaultBuildSettings = BuildSettingsVersion.V6;
|
DefaultBuildSettings = BuildSettingsVersion.V7;
|
||||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
|
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
|
||||||
ExtraModuleNames.Add("Table");
|
ExtraModuleNames.Add("Table");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
<Solution>
|
||||||
|
<Configurations>
|
||||||
|
<BuildType Name="DebugGame" />
|
||||||
|
<BuildType Name="DebugGame Editor" />
|
||||||
|
<BuildType Name="Development" />
|
||||||
|
<BuildType Name="Development Editor" />
|
||||||
|
<BuildType Name="Shipping" />
|
||||||
|
<Platform Name="Win64" />
|
||||||
|
<Platform Name="Win64-arm64" />
|
||||||
|
<Platform Name="Win64-arm64ec" />
|
||||||
|
</Configurations>
|
||||||
|
<Folder Name="/Engine/">
|
||||||
|
<Project Path="Intermediate/ProjectFiles/UE5.vcxproj" Id="99002b2b-f5da-337f-af6f-e86d1d78a0d0">
|
||||||
|
<BuildType Project="BuiltWithUnrealBuildTool" />
|
||||||
|
<Platform Project="Win64" />
|
||||||
|
<Build Project="false" />
|
||||||
|
</Project>
|
||||||
|
</Folder>
|
||||||
|
<Folder Name="/Games/">
|
||||||
|
<Project Path="Intermediate/ProjectFiles/Table.vcxproj" Id="111ae310-8395-3d64-9119-33389f569f2d">
|
||||||
|
<BuildType Solution="DebugGame Editor|Win64" Project="DebugGame_Editor" />
|
||||||
|
<BuildType Solution="DebugGame Editor|Win64-arm64" Project="Win64_arm64_DebugGame_Editor" />
|
||||||
|
<BuildType Solution="DebugGame Editor|Win64-arm64ec" Project="Win64_arm64ec_DebugGame_Editor" />
|
||||||
|
<BuildType Solution="DebugGame|Win64-arm64" Project="Win64_arm64_DebugGame" />
|
||||||
|
<BuildType Solution="DebugGame|Win64-arm64ec" Project="Win64_arm64ec_DebugGame" />
|
||||||
|
<BuildType Solution="Development Editor|Win64" Project="Development_Editor" />
|
||||||
|
<BuildType Solution="Development Editor|Win64-arm64" Project="Win64_arm64_Development_Editor" />
|
||||||
|
<BuildType Solution="Development Editor|Win64-arm64ec" Project="Win64_arm64ec_Development_Editor" />
|
||||||
|
<BuildType Solution="Development|Win64-arm64" Project="Win64_arm64_Development" />
|
||||||
|
<BuildType Solution="Development|Win64-arm64ec" Project="Win64_arm64ec_Development" />
|
||||||
|
<BuildType Solution="Shipping|Win64-arm64" Project="Win64_arm64_Shipping" />
|
||||||
|
<BuildType Solution="Shipping|Win64-arm64ec" Project="Win64_arm64ec_Shipping" />
|
||||||
|
<Platform Project="x64" />
|
||||||
|
</Project>
|
||||||
|
</Folder>
|
||||||
|
<Folder Name="/Visualizers/">
|
||||||
|
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natjmc" />
|
||||||
|
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natstepfilter" />
|
||||||
|
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natvis" />
|
||||||
|
</Folder>
|
||||||
|
<Properties Name="ddbf523f-7eb6-4887-bd51-85a714ff87eb">
|
||||||
|
<Property Name="AvailablePlatforms" Value="Win64" />
|
||||||
|
</Properties>
|
||||||
|
</Solution>
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"FileVersion": 3,
|
"FileVersion": 3,
|
||||||
"EngineAssociation": "5.7",
|
"EngineAssociation": "5.8",
|
||||||
"Category": "",
|
"Category": "",
|
||||||
"Description": "",
|
"Description": "",
|
||||||
"Modules": [
|
"Modules": [
|
||||||
@@ -22,7 +22,7 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"Name": "VisualStudioTools",
|
"Name": "VisualStudioTools",
|
||||||
"Enabled": true,
|
"Enabled": false,
|
||||||
"SupportedTargetPlatforms": [
|
"SupportedTargetPlatforms": [
|
||||||
"Win64"
|
"Win64"
|
||||||
],
|
],
|
||||||
|
|||||||
Reference in New Issue
Block a user