Compare commits
2 Commits
d53d270dee
...
c9e46eeb41
| Author | SHA1 | Date | |
|---|---|---|---|
| c9e46eeb41 | |||
| aed58897cf |
@@ -4,8 +4,6 @@
|
||||
"Component.Unreal.Debugger",
|
||||
"Component.Unreal.Ide",
|
||||
"Microsoft.Net.Component.4.6.2.TargetingPack",
|
||||
"Microsoft.VisualStudio.Component.VC.14.38.17.8.ATL",
|
||||
"Microsoft.VisualStudio.Component.VC.14.38.17.8.x86.x64",
|
||||
"Microsoft.VisualStudio.Component.VC.14.44.17.14.ATL",
|
||||
"Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64",
|
||||
"Microsoft.VisualStudio.Component.VC.Llvm.Clang",
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
<Solution>
|
||||
<Configurations>
|
||||
<BuildType Name="Analyze" />
|
||||
<BuildType Name="Debug" />
|
||||
<BuildType Name="Development" />
|
||||
<BuildType Name="Release" />
|
||||
</Configurations>
|
||||
<Folder Name="/Dependencies/">
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Analytics.Telemetry/EpicGames.Analytics.Telemetry.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build/EpicGames.Build.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph/EpicGames.BuildGraph.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core/EpicGames.Core.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde/EpicGames.Horde.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash/EpicGames.IoHash.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers/EpicGames.LogMatchers.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.MsBuild/EpicGames.MsBuild.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.OIDC/EpicGames.OIDC.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Oodle/EpicGames.Oodle.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Fixture/EpicGames.Perforce.Fixture.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed/EpicGames.Perforce.Managed.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce/EpicGames.Perforce.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.ProjectStore/EpicGames.ProjectStore.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization/EpicGames.Serialization.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UBA/EpicGames.UBA.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.UHT/EpicGames.UHT.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/UnrealBuildTool/UnrealBuildTool.csproj" />
|
||||
</Folder>
|
||||
<Folder Name="/Tests/">
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Build.Tests/EpicGames.Build.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.BuildGraph.Tests/EpicGames.BuildGraph.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Core.Tests/EpicGames.Core.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Horde.Tests/EpicGames.Horde.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.IoHash.Tests/EpicGames.IoHash.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.LogMatchers.Tests/EpicGames.LogMatchers.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Managed.Tests/EpicGames.Perforce.Managed.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Perforce.Tests/EpicGames.Perforce.Tests.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/Shared/EpicGames.Serialization.Tests/EpicGames.Serialization.Tests.csproj" />
|
||||
</Folder>
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Platforms/VisionOS/Source/Programs/AutomationTool/VisionOS.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Android/Android.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Apple/Apple.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationTool.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/AutomationUtils/AutomationUtils.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/BuildGraph/BuildGraph.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CookedEditor/CookedEditor.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/CrowdinLocalization/CrowdinLocalization.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Gauntlet/Gauntlet.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Insights/Insights.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/IOS/IOS.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Linux/Linux.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LiveLinkHub/LiveLinkHub.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Localization/Localization.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/LowLevelTests/LowLevelTests.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mac/Mac.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Microsoft/Microsoft.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Mutable/Mutable.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/OneSkyLocalization/OneSkyLocalization.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Scripts/AutomationScripts.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SmartlingLocalization/SmartlingLocalization.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/SteamDeck/SteamDeck.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Turnkey/Turnkey.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/TVOS/TVOS.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/Win/Win.Automation.csproj" />
|
||||
<Project Path="G:/Unreal/UE_5.8/Engine/Source/Programs/AutomationTool/XLocLocalization/XLocLocalization.Automation.csproj" />
|
||||
</Solution>
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,68 @@
|
||||
# Table-UE
|
||||
|
||||
> **Work in progress.** This is an actively developed personal project, not a finished product — expect incomplete features, rough edges, and breaking changes.
|
||||
|
||||
Unreal Engine side of a real-time, gesture-controlled smart tabletop for tabletop wargaming (Warhammer 40k). A downward-facing multi-camera rig tracks players' hands over the physical table; this project receives the 3D hand data over UDP, recognizes gestures, and drives an interactive projected overlay (rulers, markers, HUDs, etc.) directly on the tabletop surface.
|
||||
|
||||
The companion computer-vision/UI application that captures and streams the hand-tracking data lives in [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python).
|
||||
|
||||
## Overview
|
||||
|
||||
- Built with **Unreal Engine 5.8** (C++ gameplay module + Blueprints).
|
||||
- Receives per-frame 3D hand landmark data (21 points/hand, MediaPipe topology) via a **raw UDP socket**, decoded into `FHandPacketHand` structs.
|
||||
- Spawns and updates `AHandActor` instances per tracked hand and exposes them to Blueprints for visualization/debugging.
|
||||
- A **JSON-driven gesture binding system** matches live hand poses against configurable gesture definitions (e.g. "point with index finger") and triggers corresponding **Gesture Actions**.
|
||||
- Gesture Actions are self-contained, extendable Actors that own their own on-table visuals ("Table Icons") — e.g. a `Ruler` gesture action that draws a measuring line with tick marks between two points and displays the distance in real-world units.
|
||||
- A calibration scale factor (`UEUnitsPerUnit`) converts between real-world measurements (inches/cm on the physical table) and Unreal units, set up per session.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Table-Python (multi-camera capture, MediaPipe hand tracking, stereo triangulation)
|
||||
│ UDP (custom binary hand packet protocol)
|
||||
▼
|
||||
ADetectionManager — UDP receiver, decodes packets into hand data
|
||||
│
|
||||
▼
|
||||
AHandManager — spawns/despawns AHandActor per tracked hand id
|
||||
│
|
||||
▼
|
||||
UGestureManagerComponent — loads gesture definitions from JSON, evaluates hand poses each tick
|
||||
│
|
||||
▼
|
||||
AGestureAction (+ subclasses, e.g. AGestureAction_Ruler)
|
||||
│
|
||||
▼
|
||||
ATableIconActor (+ subclasses) — the actual on-table visuals (lines, labels, procedural meshes, UMG widgets)
|
||||
```
|
||||
|
||||
### Key classes
|
||||
|
||||
| Class | Responsibility |
|
||||
|---|---|
|
||||
| [`ADetectionManager`](Source/Table/Public/DetectionManager.h) | Owns the UDP socket, parses incoming hand-tracking packets |
|
||||
| [`AHandManager`](Source/Table/Public/HandManager.h) | Tracks the set of currently-active hands, broadcasts added/removed delegates |
|
||||
| [`AHandActor`](Source/Table/Public/HandActor.h) / [`FingerTip`](Source/Table/Public/FingerTip.h) | Represents a single tracked hand and its fingertip positions in world space |
|
||||
| [`UGestureManagerComponent`](Source/Table/Public/GestureManagerComponent.h) | Loads a JSON gesture-binding config and evaluates gestures against tracked hands |
|
||||
| [`UGestureDefinition`](Source/Table/Public/GestureDefinition.h) | Scores how well a hand pose matches a named gesture (overridden per gesture) |
|
||||
| [`AGestureAction`](Source/Table/Public/GestureAction.h) / [`AGestureAction_Ruler`](Source/Table/Public/MyGestureActions.h) | Lifecycle (init/update/end) for an active gesture, e.g. a ruler measurement |
|
||||
| [`ATableIconActor`](Source/Table/Public/TableIconActor.h) / [`ATableIconLineActor`](Source/Table/Public/MyTableIconActors.h) | Visual elements drawn on the table for the duration of a gesture |
|
||||
|
||||
Gesture-to-action bindings are configured in JSON (see `Content/WorldObjects/GestureActions/`), so new gestures can be wired up without touching C++.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Unreal Engine **5.8**
|
||||
- Visual Studio 2022 (Windows) with the "Game development with C++" workload
|
||||
- A running instance of [Table-Python](https://git.bitrux.de/Ottos_Projects/Table-Python) sending hand-tracking data over UDP (default `127.0.0.1:9000`) if you want live hand input; the level can otherwise be explored/tested without a live feed.
|
||||
|
||||
## Getting started
|
||||
|
||||
1. Clone the repo and generate project files by right-clicking `Table.uproject` → **Generate Visual Studio project files** (or open it directly in the Unreal Editor, which will prompt to build missing modules).
|
||||
2. Open `Table.sln` in Visual Studio and build, or open `Table.uproject` in Unreal Editor.
|
||||
3. Press **Play** to start the `ADetectionManager` UDP listener.
|
||||
4. Start the Python tracking application to begin streaming live hand data to the table.
|
||||
|
||||
## Status
|
||||
|
||||
Personal/hobby project, actively evolving — gesture set, table icons, and HUD content are still being expanded.
|
||||
@@ -8,8 +8,8 @@ public class TableTarget : TargetRules
|
||||
public TableTarget(TargetInfo Target) : base(Target)
|
||||
{
|
||||
Type = TargetType.Game;
|
||||
DefaultBuildSettings = BuildSettingsVersion.V6;
|
||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
|
||||
DefaultBuildSettings = BuildSettingsVersion.V7;
|
||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
|
||||
ExtraModuleNames.Add("Table");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,44 +3,62 @@
|
||||
|
||||
#include "GestureAction.h"
|
||||
#include "HandActor.h"
|
||||
#include "TableIconActor.h"
|
||||
|
||||
// Sets default values
|
||||
AGestureAction::AGestureAction()
|
||||
{
|
||||
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||
PrimaryActorTick.bCanEverTick = true;
|
||||
|
||||
PrimaryActorTick.bCanEverTick = false;
|
||||
}
|
||||
|
||||
// Called when the game starts or when spawned
|
||||
void AGestureAction::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
|
||||
}
|
||||
|
||||
// Called every frame
|
||||
void AGestureAction::Tick(float DeltaTime)
|
||||
void AGestureAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
|
||||
{
|
||||
Super::Tick(DeltaTime);
|
||||
|
||||
HandA = Hand;
|
||||
bRequiresSecondHand = Binding.LeftGesture != "" && Binding.RightGesture != "";
|
||||
bExtendable = Binding.ExtendedGestures.IsEmpty();
|
||||
Binding = NewBinding;
|
||||
HandManager = Hand->HandManager;
|
||||
UEUnitsPerUnit = NewUEUnitsPerUnit;
|
||||
}
|
||||
|
||||
void AGestureAction::UpdateAction()
|
||||
void AGestureAction::UpdateAction_Implementation()
|
||||
{
|
||||
Destroy();
|
||||
}
|
||||
|
||||
void AGestureAction::AbortAction()
|
||||
void AGestureAction::AbortAction_Implementation()
|
||||
{
|
||||
for (ATableIconActor* Icon : Icons)
|
||||
Icon->bPersistOnActionEnd = false;
|
||||
EndGesture();
|
||||
}
|
||||
|
||||
void AGestureAction::AddExtension(AHandActor* OtherHand)
|
||||
void AGestureAction::AddExtension_Implementation(AGestureAction* OtherAction)
|
||||
{
|
||||
LinkedActions = OtherAction;
|
||||
}
|
||||
|
||||
void AGestureAction::AddOtherHand_Implementation(AHandActor* OtherHand)
|
||||
{
|
||||
HandB = OtherHand;
|
||||
}
|
||||
|
||||
void AGestureAction::EndGesture()
|
||||
void AGestureAction::EndGesture_Implementation()
|
||||
{
|
||||
DestroyIcons();
|
||||
Destroy();
|
||||
}
|
||||
|
||||
void AGestureAction::DestroyIcons()
|
||||
{
|
||||
for (ATableIconActor* Icon : Icons)
|
||||
if (IsValid(Icon) ) Icon->OnActionEnded();
|
||||
Icons.Empty();
|
||||
}
|
||||
@@ -40,6 +40,11 @@ const FVector& UGestureDefinition::GetPoint(const AHandActor* Hand, int32 Index)
|
||||
return Hand->Points[Index];
|
||||
}
|
||||
|
||||
bool UGestureDefinition::IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold) const
|
||||
{
|
||||
return GetFingerExtendedScore(Hand, Finger) > Threshold;
|
||||
}
|
||||
|
||||
float UGestureDefinition::GetHandScale(const AHandActor* Hand) const
|
||||
{
|
||||
const TArray<FVector>& P = Hand->Points;
|
||||
@@ -53,30 +58,44 @@ float UGestureDefinition::GetNormalizedDistance(const AHandActor* Hand, int32 A,
|
||||
}
|
||||
float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const
|
||||
{
|
||||
float Dist = GetNormalizedDistance(
|
||||
Hand,
|
||||
GetFingerTipIndex(A),
|
||||
GetFingerTipIndex(B)
|
||||
);
|
||||
const FVector& P1 = Hand->Points[GetFingerTipIndex(A)];
|
||||
const FVector& P2 = Hand->Points[GetFingerTipIndex(B)];
|
||||
|
||||
float Dist = FVector::Dist(P1, P2);
|
||||
|
||||
float HandScale = FVector::Dist( Hand->Points[0], Hand->Points[9] );
|
||||
|
||||
float Normalized = Dist / HandScale;
|
||||
|
||||
//UE_LOG(LogTemp, Warning, TEXT("Dist: %.3f | HandScale: %.3f | Norm: %.3f"), Dist, HandScale, Normalized);
|
||||
|
||||
const float Threshold = 0.25f;
|
||||
|
||||
return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f);
|
||||
return 1.f - FMath::Clamp( Normalized / Threshold, 0.f, 1.f );
|
||||
}
|
||||
|
||||
float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const
|
||||
{
|
||||
const TArray<FVector>& P = Hand->Points;
|
||||
|
||||
if (Finger == EHandFinger::Thumb)
|
||||
{
|
||||
// Thumb: compare tip->base against index base->wrist as reference direction
|
||||
const FVector& Base = P[2];
|
||||
const FVector& Tip = P[4];
|
||||
const FVector& Ref = P[5]; // index base as stable reference
|
||||
float Dot = FVector::DotProduct(
|
||||
(Tip - Base).GetSafeNormal(),
|
||||
(Ref - P[0]).GetSafeNormal());
|
||||
return FMath::Clamp(Dot, 0.f, 1.f);
|
||||
}
|
||||
|
||||
const FVector& Wrist = P[0];
|
||||
const FVector& Base = P[GetFingerBaseIndex(Finger)];
|
||||
const FVector& Tip = P[GetFingerTipIndex(Finger)];
|
||||
|
||||
float Dot = FVector::DotProduct(
|
||||
(Tip - Base).GetSafeNormal(),
|
||||
(Base - Wrist).GetSafeNormal()
|
||||
);
|
||||
|
||||
(Base - Wrist).GetSafeNormal());
|
||||
return FMath::Clamp(Dot, 0.f, 1.f);
|
||||
}
|
||||
|
||||
@@ -91,9 +110,7 @@ float UGestureDefinition::GetFingersAverage(const AHandActor* Hand, const TArray
|
||||
|
||||
for (EHandFinger Finger : Fingers)
|
||||
{
|
||||
Sum += bExtended
|
||||
? GetFingerExtendedScore(Hand, Finger)
|
||||
: GetFingerCurledScore(Hand, Finger);
|
||||
Sum += bExtended ? GetFingerExtendedScore(Hand, Finger) : GetFingerCurledScore(Hand, Finger);
|
||||
}
|
||||
|
||||
return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f;
|
||||
@@ -107,19 +124,57 @@ FVector UGestureDefinition::GetFingerDirection(const AHandActor* Hand, EHandFing
|
||||
return (Tip - Base).GetSafeNormal();
|
||||
}
|
||||
|
||||
float UGesture_PinchPoint::Evaluate(AHandActor* Hand) const
|
||||
float UGesture_PinchIndex::Evaluate(AHandActor* Hand) const
|
||||
{
|
||||
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
|
||||
float IndexExtended = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||
|
||||
float OthersCurled = GetFingersAverage(
|
||||
Hand,
|
||||
{ EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky },
|
||||
false
|
||||
);
|
||||
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky }, false );
|
||||
|
||||
return
|
||||
Pinch * 0.5f +
|
||||
IndexExtended * 0.3f +
|
||||
OthersCurled * 0.2f;
|
||||
return Pinch * 0.5f + IndexExtended * 0.3f + OthersCurled * 0.2f;
|
||||
}
|
||||
|
||||
float UGesture_PinchMiddleIndex::Evaluate(AHandActor* Hand) const
|
||||
{
|
||||
float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Middle);
|
||||
Pinch += GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
|
||||
|
||||
float MiddleExtended = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||
float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Ring, EHandFinger::Pinky }, false );
|
||||
|
||||
return Pinch * 0.5f + MiddleExtended * 0.3f + OthersCurled * 0.2f;
|
||||
|
||||
}
|
||||
|
||||
float UGesture_PointIndex::Evaluate(AHandActor* Hand) const
|
||||
{
|
||||
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||
float Middle = GetFingerCurledScore(Hand, EHandFinger::Middle);
|
||||
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||
|
||||
return ( Index + Middle + Ring + Pinky + Thumb ) / 5.f;
|
||||
}
|
||||
|
||||
float UGesture_PointMiddle::Evaluate(AHandActor* Hand) const
|
||||
{
|
||||
float Index = GetFingerCurledScore(Hand, EHandFinger::Index);
|
||||
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||
|
||||
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
|
||||
}
|
||||
|
||||
float UGesture_PointIndexMiddle::Evaluate(AHandActor* Hand) const
|
||||
{
|
||||
float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
|
||||
float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
|
||||
float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
|
||||
float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
|
||||
float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
|
||||
|
||||
return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ UGestureManagerComponent::UGestureManagerComponent()
|
||||
// off to improve performance if you don't need them.
|
||||
PrimaryComponentTick.bCanEverTick = false;
|
||||
|
||||
HandManager = Cast<AHandManager>(GetOwner());
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ void UGestureManagerComponent::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
|
||||
HandManager = Cast<AHandManager>(GetOwner());
|
||||
|
||||
HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded);
|
||||
HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved);
|
||||
|
||||
@@ -42,17 +44,16 @@ void UGestureManagerComponent::EvaluateHands()
|
||||
if (Hand->bUsedThisTick) continue;
|
||||
switch (Hand->Phase)
|
||||
{
|
||||
case EGesturePhase::Started:
|
||||
case EHandGesturePhase::Started:
|
||||
TriggerAction(Hand);
|
||||
break;
|
||||
case EGesturePhase::Ended:
|
||||
case EHandGesturePhase::Ended:
|
||||
Hand->ResetGesture();
|
||||
EndGesture();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
UpdateOngoingAction();
|
||||
UpdateOngoingActions();
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
|
||||
@@ -62,14 +63,16 @@ void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
|
||||
|
||||
void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand)
|
||||
{
|
||||
Hand->PairedHand->PairedHand = nullptr;
|
||||
Hand->ResetGesture();
|
||||
if (!Hand->ActiveAction) return;
|
||||
if (!ActiveGestureActions.Contains(Hand->ActiveAction)) return;
|
||||
Hand->ActiveAction->Destroy();
|
||||
ActiveGestureActions.Remove(Hand->ActiveAction);
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::LoadBindings(const FString& FilePath)
|
||||
{
|
||||
FString JsonString;
|
||||
|
||||
FString JsonString = "";
|
||||
FString Temp = "";
|
||||
if (!FFileHelper::LoadFileToString(JsonString, *FilePath))
|
||||
{
|
||||
return;
|
||||
@@ -92,9 +95,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
|
||||
|
||||
FGestureBinding Binding;
|
||||
|
||||
Obj->TryGetStringField(TEXT("action"), Binding.ActionName);
|
||||
Obj->TryGetStringField(TEXT("left"), Binding.LeftGesture);
|
||||
Obj->TryGetStringField(TEXT("right"), Binding.RightGesture);
|
||||
Obj->TryGetStringField(TEXT("action"), Temp);
|
||||
Binding.ActionName = FName(Temp);
|
||||
Obj->TryGetStringField(TEXT("left"), Temp);
|
||||
Binding.LeftGesture = FName(Temp);
|
||||
Obj->TryGetStringField(TEXT("right"), Temp);
|
||||
Binding.RightGesture = FName(Temp);
|
||||
|
||||
const TArray<TSharedPtr<FJsonValue>>* ExtArray;
|
||||
if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray))
|
||||
@@ -106,10 +112,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
|
||||
|
||||
FExtendedGestureBinding Ext;
|
||||
|
||||
ExtObj->TryGetStringField(TEXT("gesture"), Ext.GestureName);
|
||||
ExtObj->TryGetStringField(TEXT("action"), Ext.GestureAction);
|
||||
ExtObj->TryGetStringField(TEXT("gesture"), Temp);
|
||||
Ext.GestureName = FName(Temp);
|
||||
ExtObj->TryGetStringField(TEXT("action"), Temp);
|
||||
Ext.GestureAction = FName(Temp);
|
||||
|
||||
Binding.ExentdedGestures.Add(Ext);
|
||||
Binding.ExtendedGestures.Add(Ext);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -143,6 +151,7 @@ void UGestureManagerComponent::BuildDefinitionMap()
|
||||
UGestureDefinition* Def = NewObject<UGestureDefinition>(this, Row->DefinitionClass);
|
||||
|
||||
DefinitionMap.Add(Def->GestureName, Def);
|
||||
GestureDefinitions.Add(Def);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,176 +168,171 @@ void UGestureManagerComponent::BuildActionMap()
|
||||
{
|
||||
if (!Row || !Row->ActionClass) continue;
|
||||
|
||||
ActionClassMap.Add(
|
||||
Row->ActionClass->GetName(),
|
||||
Row->ActionClass
|
||||
|
||||
);
|
||||
ActionClassMap.Add( FName(Row->ActionClass->GetName()), Row->ActionClass );
|
||||
}
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::CheckGestureDefinitions(TArray<AHandActor*>& Hands)
|
||||
{
|
||||
const float MinScoreThreshold = 0.5f;
|
||||
//const float MinScoreThreshold = 0.7f;
|
||||
|
||||
|
||||
for (AHandActor* Hand : Hands) {
|
||||
float Threshold = (Hand->Phase == EHandGesturePhase::Ongoing || Hand->Phase == EHandGesturePhase::Started) ? 0.60f : 0.75f;
|
||||
|
||||
float BestScore = 0.f;
|
||||
FString BestGesture = "";
|
||||
FName BestGesture = "";
|
||||
|
||||
for (UGestureDefinition* Gesture : GestureDefinitions) {
|
||||
|
||||
float Score = Gesture->Evaluate(Hand);
|
||||
float Score = Gesture->Evaluate(Hand); // Getting the Score of the gestures to check
|
||||
|
||||
if (Score > BestScore)
|
||||
//UE_LOG(LogTemp, Warning, TEXT("%s -> %s : %.3f"), *Hand->GetName(), *Gesture->GestureName, Score);
|
||||
|
||||
if (Score > BestScore) // Getting the gesture with the highest Score
|
||||
{
|
||||
BestScore = Score;
|
||||
BestGesture = Gesture->GestureName;
|
||||
}
|
||||
}
|
||||
|
||||
if (BestScore < MinScoreThreshold)
|
||||
// Check if the gesture with the highest score is a valid Gesture
|
||||
if (BestScore < Threshold)
|
||||
{
|
||||
if (Hand->Phase != EGesturePhase::None)
|
||||
if (Hand->Phase != EHandGesturePhase::None)
|
||||
{
|
||||
Hand->FailFrames++;
|
||||
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] Below threshold (%s) Score=%.3f Fail=%d/%d"),
|
||||
*Hand->GetName(),
|
||||
*Hand->ActiveGestureName.ToString(),
|
||||
BestScore,
|
||||
Hand->FailFrames,
|
||||
FailThreshold);
|
||||
//if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
|
||||
if (Hand->FailFrames >= FailThreshold)
|
||||
{
|
||||
Hand->Phase = EGesturePhase::Ended;
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] -> ENDED (%s)"),
|
||||
*Hand->GetName(),
|
||||
*Hand->ActiveGestureName.ToString());
|
||||
|
||||
Hand->Phase = EHandGesturePhase::Ended;
|
||||
}
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle valid Gesture
|
||||
switch (Hand->Phase)
|
||||
{
|
||||
case EGesturePhase::None:
|
||||
case EHandGesturePhase::None: // Starting a Candidate
|
||||
Hand->ActiveGestureName = BestGesture;
|
||||
Hand->Phase = EGesturePhase::Candidate;
|
||||
Hand->Phase = EHandGesturePhase::Candidate;
|
||||
Hand->CandidateFrames = 1;
|
||||
Hand->FailFrames = 0;
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] NONE -> CANDIDATE (%s)"),
|
||||
*Hand->GetName(),
|
||||
*BestGesture.ToString());
|
||||
|
||||
break;
|
||||
case EGesturePhase::Candidate:
|
||||
if (Hand->ActiveGestureName != BestGesture)
|
||||
case EHandGesturePhase::Candidate: // Checking if the Candidate stays valid or another gesture is made
|
||||
if (Hand->ActiveGestureName == BestGesture)
|
||||
{
|
||||
Hand->FailFrames++;
|
||||
Hand->CandidateFrames++;
|
||||
Hand->FailFrames = 0;
|
||||
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] Candidate %s (%d/%d)"),
|
||||
*Hand->GetName(),
|
||||
*BestGesture.ToString(),
|
||||
Hand->CandidateFrames,
|
||||
CandidateThreshold);
|
||||
|
||||
//if (Hand->CandidateFrames >= CandidateThreshold) Hand->Phase = EHandGesturePhase::Started;
|
||||
if (Hand->CandidateFrames >= CandidateThreshold)
|
||||
{
|
||||
Hand->Phase = EHandGesturePhase::Started;
|
||||
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] CANDIDATE -> STARTED (%s)"),
|
||||
*Hand->GetName(),
|
||||
*BestGesture.ToString());
|
||||
}
|
||||
break;
|
||||
}
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] Candidate switched %s -> %s"),
|
||||
*Hand->GetName(),
|
||||
*Hand->ActiveGestureName.ToString(),
|
||||
*BestGesture.ToString());
|
||||
|
||||
Hand->CandidateFrames++;
|
||||
Hand->ActiveGestureName = BestGesture;
|
||||
Hand->CandidateFrames = 1;
|
||||
Hand->FailFrames = 0;
|
||||
|
||||
if (Hand->CandidateFrames >= CandidateThreshold)
|
||||
Hand->Phase = EGesturePhase::Started;
|
||||
|
||||
break;
|
||||
case EGesturePhase::Ongoing:
|
||||
case EHandGesturePhase::Ongoing: // Checking if the gesture is still ongoing
|
||||
//if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
|
||||
//else Hand->FailFrames = 0;
|
||||
if (Hand->ActiveGestureName != BestGesture)
|
||||
{
|
||||
Hand->FailFrames++;
|
||||
|
||||
if (Hand->FailFrames >= FailThreshold)
|
||||
{
|
||||
Hand->Phase = EGesturePhase::Ended;
|
||||
}
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] Ongoing fail %s (%d/%d)"),
|
||||
*Hand->GetName(),
|
||||
*Hand->ActiveGestureName.ToString(),
|
||||
Hand->FailFrames,
|
||||
FailThreshold);
|
||||
}
|
||||
else
|
||||
{
|
||||
Hand->FailFrames = 0;
|
||||
if (Hand->FailFrames > 0)
|
||||
{
|
||||
UE_LOG(LogTemp, Warning,
|
||||
TEXT("[%s] Ongoing recovered %s"),
|
||||
*Hand->GetName(),
|
||||
*Hand->ActiveGestureName.ToString());
|
||||
}
|
||||
}
|
||||
break;
|
||||
//case EHandGesturePhase::Started:
|
||||
// break;
|
||||
default: // If the gesture would be stuck somewhere
|
||||
if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
|
||||
|
||||
if (Hand->FailFrames >= FailThreshold) Hand->ResetGesture(); // This optimaly shouldnt be called here
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
|
||||
if (Hand->FailFrames >= FailThreshold)
|
||||
Hand->Phase = EGesturePhase::Ended;
|
||||
// Checking if a gesture has ended
|
||||
if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
|
||||
}
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::TriggerAction(AHandActor* Hand)
|
||||
{
|
||||
if (!Hand->bUsedThisTick) return;
|
||||
if (Hand->bUsedThisTick) return;
|
||||
|
||||
if (Hand->IsPaired())
|
||||
{
|
||||
AHandActor* Other = Hand->PairedHand;
|
||||
|
||||
for (const FGestureBinding& Binding : GestureBindings)
|
||||
{
|
||||
const bool bLeftMatch = (Hand->Handedness == EHandedness::Left && Hand->ActiveGestureName == Binding.LeftGesture &&
|
||||
Other->ActiveGestureName == Binding.RightGesture);
|
||||
|
||||
const bool bRightMatch = (Hand->Handedness == EHandedness::Right && Hand->ActiveGestureName == Binding.RightGesture &&
|
||||
Other->ActiveGestureName == Binding.LeftGesture);
|
||||
|
||||
if (!(bLeftMatch || bRightMatch))
|
||||
continue;
|
||||
|
||||
// both must be ready
|
||||
if (Other->Phase != EGesturePhase::Started)
|
||||
continue;
|
||||
|
||||
// mark used
|
||||
Hand->bUsedThisTick = true;
|
||||
Other->bUsedThisTick = true;
|
||||
|
||||
Hand->Phase = EGesturePhase::Ongoing;
|
||||
Other->Phase = EGesturePhase::Ongoing;
|
||||
|
||||
// spawn action (centralized factory recommended)
|
||||
SpawnGestureAction(Binding.ActionName, Hand, Other);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (TryExtendGestureAction(Hand)) return;
|
||||
|
||||
for (const FGestureBinding& Binding : GestureBindings)
|
||||
{
|
||||
const bool bMatch =
|
||||
(Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName) ||
|
||||
(Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
||||
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|
||||
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
||||
|
||||
if (!bMatch)
|
||||
continue;
|
||||
if (!bMatches) continue;
|
||||
|
||||
// if paired hand is actively running something, allow extension check later
|
||||
if (Hand->IsPaired() && Hand->PairedHand && Hand->PairedHand->Phase == EGesturePhase::Ongoing)
|
||||
{
|
||||
for (const FExtendedGestureBinding& Ext : Binding.ExentdedGestures)
|
||||
{
|
||||
if (Ext.GestureName != Hand->ActiveGestureName)
|
||||
continue;
|
||||
AGestureAction* NewAction = SpawnGestureAction(Binding.ActionName, Hand);
|
||||
NewAction->InitAction(Hand, Binding, UEUnitsPerUnit);
|
||||
|
||||
if (Ext.GestureAction == Binding.ActionName)
|
||||
{
|
||||
// extend current action
|
||||
ExtendGestureAction(Binding.ActionName, Hand);
|
||||
Hand->bUsedThisTick = true;
|
||||
return;
|
||||
}
|
||||
else
|
||||
{
|
||||
// branch: spawn new action linked to existing one
|
||||
SpawnGestureActionWithParent(Ext.GestureAction, Hand, Hand->PairedHand);
|
||||
Hand->bUsedThisTick = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normal one-hand action
|
||||
Hand->bUsedThisTick = true;
|
||||
SpawnGestureAction(Binding.ActionName, Hand, nullptr);
|
||||
return;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
void UGestureManagerComponent::UpdateOngoingAction()
|
||||
void UGestureManagerComponent::UpdateOngoingActions()
|
||||
{
|
||||
for (AGestureAction* Action : ActiveGestureActions)
|
||||
{
|
||||
@@ -338,18 +342,94 @@ void UGestureManagerComponent::UpdateOngoingAction()
|
||||
}
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B)
|
||||
AGestureAction* UGestureManagerComponent::SpawnGestureAction(FName ActionName, AHandActor* Hand)
|
||||
{
|
||||
|
||||
TSubclassOf<AGestureAction>* ActionClassPtr = ActionClassMap.Find(ActionName);
|
||||
|
||||
if (!ActionClassPtr || !Hand) return nullptr;
|
||||
|
||||
FActorSpawnParameters SpawnParams;
|
||||
SpawnParams.Owner = GetOwner();
|
||||
SpawnParams.SpawnCollisionHandlingOverride =
|
||||
ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
|
||||
|
||||
AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
|
||||
*ActionClassPtr,
|
||||
FVector::ZeroVector,
|
||||
FRotator::ZeroRotator,
|
||||
SpawnParams);
|
||||
|
||||
if (!NewAction) return nullptr;
|
||||
|
||||
|
||||
Hand->ActiveAction = NewAction;
|
||||
Hand->bUsedThisTick = true;
|
||||
Hand->Phase = EHandGesturePhase::Ongoing;
|
||||
|
||||
return NewAction;
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::ExtendGestureAction(FString ActionName, AHandActor* Hand)
|
||||
bool UGestureManagerComponent::TryExtendGestureAction(AHandActor* Hand)
|
||||
{
|
||||
if (!Hand->IsPaired() || Hand->PairedHand->Phase != EHandGesturePhase::Ongoing || !Hand->PairedHand->ActiveAction) return false;
|
||||
|
||||
if (TryJoinExistingAction(Hand)) return true;
|
||||
if (TryCreateExtension(Hand)) return true;
|
||||
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void UGestureManagerComponent::SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand)
|
||||
bool UGestureManagerComponent::TryJoinExistingAction(AHandActor* Hand)
|
||||
{
|
||||
AHandActor* OtherHand = Hand->PairedHand;
|
||||
AGestureAction* OtherAction = OtherHand->ActiveAction;
|
||||
|
||||
if (!OtherAction->bRequiresSecondHand) return false; // Check if its a Two Handed Action or an extension
|
||||
|
||||
FGestureBinding& Binding = OtherAction->Binding;
|
||||
|
||||
bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
|
||||
|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
|
||||
|
||||
if (!bMatches) return false;
|
||||
|
||||
OtherAction->AddOtherHand(Hand);
|
||||
OtherHand->bUsedThisTick = true;
|
||||
Hand->bUsedThisTick = true;
|
||||
Hand->Phase = EHandGesturePhase::Ongoing;
|
||||
Hand->ActiveAction = OtherAction;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool UGestureManagerComponent::TryCreateExtension(AHandActor* Hand)
|
||||
{
|
||||
AHandActor* OtherHand = Hand->PairedHand;
|
||||
AGestureAction* OtherAction = OtherHand->ActiveAction;
|
||||
|
||||
if (!OtherHand->ActiveAction->bExtendable) return false;
|
||||
|
||||
FName ActionName;
|
||||
for (const FExtendedGestureBinding& Gesture : OtherAction->Binding.ExtendedGestures)
|
||||
{
|
||||
if (Gesture.GestureName == Hand->ActiveGestureName)
|
||||
{
|
||||
ActionName = Gesture.GestureAction;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (ActionName.IsNone()) return false;
|
||||
|
||||
AGestureAction* NewAction = SpawnGestureAction(ActionName, Hand);
|
||||
|
||||
NewAction->InitAction(Hand, OtherHand->ActiveAction->Binding, UEUnitsPerUnit);
|
||||
OtherHand->bUsedThisTick = true;
|
||||
Hand->bUsedThisTick = true;
|
||||
OtherAction->AddExtension(NewAction);
|
||||
NewAction->AddExtension(OtherHand->ActiveAction);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "HandManager.h"
|
||||
#include "FingerTip.h"
|
||||
#include "DrawDebugHelpers.h"
|
||||
#include "GestureAction.h"
|
||||
|
||||
|
||||
static const int32 HandBones[][2] = {
|
||||
@@ -31,14 +32,21 @@ AHandActor::AHandActor()
|
||||
|
||||
}
|
||||
|
||||
void AHandActor::ResetGesture()
|
||||
{
|
||||
Phase = EHandGesturePhase::None;
|
||||
ActiveGestureName = "";
|
||||
CandidateFrames = 0;
|
||||
FailFrames = 0;
|
||||
if (IsValid(ActiveAction)) ActiveAction->EndGesture();
|
||||
ActiveAction = nullptr;
|
||||
}
|
||||
|
||||
// Called when the game starts or when spawned
|
||||
void AHandActor::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
GetWorld()->GetTimerManager().SetTimer(DeathTimer, this, &AHandActor::HandDeath, DeathTime, true);
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
// Called every frame
|
||||
@@ -123,8 +131,6 @@ void AHandActor::UpdateHand(const FHandPacketHand& NewHand)
|
||||
Confidence = NewHand.Confidence;
|
||||
HandId = NewHand.HandId;
|
||||
|
||||
//UE_LOG(LogTemp, Error, TEXT("Update Hand"));
|
||||
|
||||
for (int32 n = 0; n < 5; n++)
|
||||
{
|
||||
FVector f = Points[FingerTipBones[n]];
|
||||
|
||||
@@ -10,7 +10,7 @@ AHandManager::AHandManager()
|
||||
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||
PrimaryActorTick.bCanEverTick = true;
|
||||
GestureManager = CreateDefaultSubobject<UGestureManagerComponent>(TEXT("GestureManager"));
|
||||
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
|
||||
|
||||
}
|
||||
|
||||
// Called when the game starts or when spawned
|
||||
@@ -18,6 +18,8 @@ void AHandManager::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
|
||||
GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
|
||||
|
||||
if (!IsValid(HandActorClass))
|
||||
{
|
||||
HandActorClass = AHandActor::StaticClass();
|
||||
@@ -69,7 +71,7 @@ void AHandManager::AddDeadHand(AHandActor* DeadHand)
|
||||
DeadHands.Add(DeadHand);
|
||||
}
|
||||
|
||||
TArray<AHandActor*> AHandManager::GetHandList()
|
||||
TArray<AHandActor*>& AHandManager::GetHandList()
|
||||
{
|
||||
return HandList;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
|
||||
#include "MyGestureActions.h"
|
||||
#include "HandActor.h"
|
||||
#include "TableIconActor.h"
|
||||
|
||||
|
||||
void ARulerAction::UpdateAction_Implementation()
|
||||
{
|
||||
if (!HandA) return;
|
||||
|
||||
// Update end point to current fingertip
|
||||
Anchors.Add("End", HandA->Points[8]);
|
||||
|
||||
FVector Start = Anchors["Start"];
|
||||
FVector End = Anchors["End"];
|
||||
|
||||
float RawDist = FVector::Dist(Start, End);
|
||||
float Inches = GetSnappedInches(RawDist);
|
||||
|
||||
// Move line icon between start and end
|
||||
if (Icons.IsValidIndex(0) && IsValid(Icons[0]))
|
||||
{
|
||||
Icons[0]->SetActorLocation((Start + End) * 0.5f);
|
||||
// You'll add a SetEndpoints(Start, End) function to ATableIconLineActor later
|
||||
}
|
||||
|
||||
// Move label icon to midpoint, slightly above table
|
||||
if (Icons.IsValidIndex(1) && IsValid(Icons[1]))
|
||||
{
|
||||
FVector Mid = (Start + End) * 0.5f + FVector(0, 0, 5.f);
|
||||
Icons[1]->SetActorLocation(Mid);
|
||||
// Icons[1]->SetLabelText(FString::Printf(TEXT("%d\""), (int32)Inches));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
void ARulerAction::AbortAction_Implementation()
|
||||
{
|
||||
Super::AbortAction();
|
||||
}
|
||||
|
||||
void ARulerAction::AddExtension_Implementation(AGestureAction* OtherAction)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
void ARulerAction::EndGesture_Implementation()
|
||||
{
|
||||
Super::EndGesture();
|
||||
}
|
||||
|
||||
void ARulerAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
|
||||
{
|
||||
Super::InitAction(Hand, NewBinding, NewUEUnitsPerUnit);
|
||||
Anchors.Add("Start", Hand->Points[8]);
|
||||
|
||||
FActorSpawnParameters SpawnParams;
|
||||
SpawnParams.Owner = GetOwner();
|
||||
SpawnParams.SpawnCollisionHandlingOverride =
|
||||
ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
|
||||
|
||||
AGestureAction* NewAction = nullptr;
|
||||
/*AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
|
||||
*ActionClassPtr,
|
||||
FVector::ZeroVector,
|
||||
FRotator::ZeroRotator,
|
||||
SpawnParams);
|
||||
*/
|
||||
if (!NewAction) return;
|
||||
|
||||
|
||||
}
|
||||
|
||||
float ARulerAction::GetSnappedInches(float RawDistance) const
|
||||
{
|
||||
if (UEUnitsPerUnit <= 0.f) return 0.f;
|
||||
float Unit = RawDistance / UEUnitsPerUnit;
|
||||
return FMath::RoundToFloat(Unit); // snap to whole inches
|
||||
}
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
|
||||
#include "MyTableIconActors.h"
|
||||
#include "KismetProceduralMeshLibrary.h"
|
||||
#include "TableIconActor.h"
|
||||
|
||||
#include "Kismet/GameplayStatics.h"
|
||||
#include "Camera/CameraActor.h"
|
||||
|
||||
ATableIconLineActor::ATableIconLineActor()
|
||||
{
|
||||
MeshComponent = CreateDefaultSubobject<UProceduralMeshComponent>(TEXT("LineMesh"));
|
||||
MeshComponent->SetupAttachment(RootComponent);
|
||||
MeshComponent->SetCastShadow(false);
|
||||
}
|
||||
|
||||
void ATableIconLineActor::SetEndpoints(const FVector& Start, const FVector& End)
|
||||
{
|
||||
// Skip rebuild if nothing meaningful changed
|
||||
if (Start.Equals(LastStart, 0.1f) && End.Equals(LastEnd, 0.1f)) return;
|
||||
LastStart = Start;
|
||||
LastEnd = End;
|
||||
|
||||
BuildLineMesh(Start, End);
|
||||
if (bShowTickMarks) BuildTickMesh(Start, End);
|
||||
}
|
||||
|
||||
void ATableIconLineActor::BuildLineMesh(const FVector& Start, const FVector& End)
|
||||
{
|
||||
FVector Dir = (End - Start).GetSafeNormal();
|
||||
FVector Up = FVector::UpVector;
|
||||
FVector Right = FVector::CrossProduct(Dir, Up).GetSafeNormal() * LineThickness * 0.5f;
|
||||
|
||||
// A flat quad along the line, facing up
|
||||
TArray<FVector> Vertices;
|
||||
TArray<int32> Triangles;
|
||||
TArray<FVector> Normals;
|
||||
TArray<FVector2D> UVs;
|
||||
TArray<FColor> Colors;
|
||||
TArray<FProcMeshTangent> Tangents;
|
||||
|
||||
Vertices.Add(Start - Right); // 0
|
||||
Vertices.Add(Start + Right); // 1
|
||||
Vertices.Add(End + Right); // 2
|
||||
Vertices.Add(End - Right); // 3
|
||||
|
||||
Triangles = { 0, 1, 2, 0, 2, 3 };
|
||||
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
Normals.Add(FVector::UpVector);
|
||||
UVs.Add(FVector2D(i < 2 ? 0.f : 1.f, i % 2 == 0 ? 0.f : 1.f));
|
||||
Colors.Add(LineColor.ToFColor(true));
|
||||
Tangents.Add(FProcMeshTangent(Dir, false));
|
||||
}
|
||||
|
||||
// Section 0 = main line
|
||||
MeshComponent->CreateMeshSection(0, Vertices, Triangles, Normals, UVs,
|
||||
Colors, Tangents, false);
|
||||
}
|
||||
|
||||
void ATableIconLineActor::BuildTickMesh(const FVector& Start, const FVector& End)
|
||||
{
|
||||
if (UEUnitsPerUnit <= 0.f || TickInterval <= 0.f) return;
|
||||
|
||||
float TotalDist = FVector::Dist(Start, End);
|
||||
float TickSpacing = TickInterval * UEUnitsPerUnit;
|
||||
int32 NumTicks = FMath::FloorToInt(TotalDist / TickSpacing);
|
||||
|
||||
if (NumTicks <= 0)
|
||||
{
|
||||
MeshComponent->ClearMeshSection(1);
|
||||
return;
|
||||
}
|
||||
|
||||
FVector Dir = (End - Start).GetSafeNormal();
|
||||
FVector Right = FVector::CrossProduct(Dir, FVector::UpVector).GetSafeNormal();
|
||||
|
||||
TArray<FVector> Vertices;
|
||||
TArray<int32> Triangles;
|
||||
TArray<FVector> Normals;
|
||||
TArray<FVector2D> UVs;
|
||||
TArray<FColor> Colors;
|
||||
TArray<FProcMeshTangent> Tangents;
|
||||
|
||||
float TickHalfLen = LineThickness * 3.f;
|
||||
float TickHalfW = LineThickness * 0.5f;
|
||||
|
||||
for (int32 i = 1; i <= NumTicks; i++)
|
||||
{
|
||||
FVector Center = Start + Dir * (TickSpacing * i);
|
||||
FVector A = Center - Right * TickHalfLen - Dir * TickHalfW;
|
||||
FVector B = Center + Right * TickHalfLen - Dir * TickHalfW;
|
||||
FVector C = Center + Right * TickHalfLen + Dir * TickHalfW;
|
||||
FVector D = Center - Right * TickHalfLen + Dir * TickHalfW;
|
||||
|
||||
int32 Base = Vertices.Num();
|
||||
Vertices.Add(A); Vertices.Add(B);
|
||||
Vertices.Add(C); Vertices.Add(D);
|
||||
|
||||
Triangles.Add(Base); Triangles.Add(Base + 1); Triangles.Add(Base + 2);
|
||||
Triangles.Add(Base); Triangles.Add(Base + 2); Triangles.Add(Base + 3);
|
||||
|
||||
for (int j = 0; j < 4; j++)
|
||||
{
|
||||
Normals.Add(FVector::UpVector);
|
||||
UVs.Add(FVector2D::ZeroVector);
|
||||
Colors.Add(LineColor.ToFColor(true));
|
||||
Tangents.Add(FProcMeshTangent(Dir, false));
|
||||
}
|
||||
}
|
||||
|
||||
// Section 1 = tick marks
|
||||
MeshComponent->CreateMeshSection(1, Vertices, Triangles, Normals, UVs,
|
||||
Colors, Tangents, false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
ATableIconWidgetActor::ATableIconWidgetActor()
|
||||
{
|
||||
PrimaryActorTick.bCanEverTick = true;
|
||||
|
||||
WidgetComponent = CreateDefaultSubobject<UWidgetComponent>(TEXT("WidgetComponent"));
|
||||
WidgetComponent->SetupAttachment(RootComponent);
|
||||
WidgetComponent->SetDrawAtDesiredSize(true);
|
||||
WidgetComponent->SetWidgetSpace(EWidgetSpace::World);
|
||||
WidgetComponent->SetCollisionEnabled(ECollisionEnabled::NoCollision);
|
||||
|
||||
WidgetInteraction = CreateDefaultSubobject<UWidgetInteractionComponent>(TEXT("WidgetInteraction"));
|
||||
WidgetInteraction->SetupAttachment(RootComponent);
|
||||
WidgetInteraction->InteractionSource = EWidgetInteractionSource::Custom;
|
||||
WidgetInteraction->InteractionDistance = 10.f;
|
||||
}
|
||||
|
||||
void ATableIconWidgetActor::Tick(float DeltaTime)
|
||||
{
|
||||
Super::Tick(DeltaTime);
|
||||
|
||||
if (!bFaceCamera) return;
|
||||
|
||||
APlayerCameraManager* CamManager = UGameplayStatics::GetPlayerCameraManager(this, 0);
|
||||
if (!CamManager) return;
|
||||
|
||||
FVector CamLocation = CamManager->GetCameraLocation();
|
||||
FVector ToCamera = (CamLocation - GetActorLocation()).GetSafeNormal();
|
||||
|
||||
// Only rotate on yaw — keeps widget flat relative to table
|
||||
ToCamera.Z = 0.f;
|
||||
if (!ToCamera.IsNearlyZero())
|
||||
SetActorRotation(ToCamera.ToOrientationRotator());
|
||||
}
|
||||
|
||||
void ATableIconWidgetActor::SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass)
|
||||
{
|
||||
if (!InWidgetClass) return;
|
||||
WidgetComponent->SetWidgetClass(InWidgetClass);
|
||||
WidgetComponent->InitWidget();
|
||||
}
|
||||
|
||||
UUserWidget* ATableIconWidgetActor::GetWidget() const
|
||||
{
|
||||
return WidgetComponent->GetUserWidgetObject();
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
|
||||
#include "TableIconActor.h"
|
||||
|
||||
// Sets default values
|
||||
ATableIconActor::ATableIconActor()
|
||||
{
|
||||
// Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it.
|
||||
PrimaryActorTick.bCanEverTick = false;
|
||||
|
||||
}
|
||||
|
||||
// Called when the game starts or when spawned
|
||||
void ATableIconActor::BeginPlay()
|
||||
{
|
||||
Super::BeginPlay();
|
||||
|
||||
}
|
||||
|
||||
void ATableIconActor::OnActionEnded_Implementation()
|
||||
{
|
||||
if (!bPersistOnActionEnd) Destroy();
|
||||
}
|
||||
|
||||
void ATableIconActor::OnActionUpdated_Implementation()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -107,8 +107,8 @@ FContentSource UUserHUDWidget::ParseContentSource(TSharedPtr<FJsonObject> Source
|
||||
|
||||
if (ContentTypeString == TEXT("Image")) Source.ContentType = EContentSourceType::Image;
|
||||
else if (ContentTypeString == TEXT("PDF")) Source.ContentType = EContentSourceType::PDF;
|
||||
else if (ContentTypeString == TEXT("TextFile")) Source.ContentType = EContentSourceType::TextFile;
|
||||
else if (ContentTypeString == TEXT("InlineText")) Source.ContentType = EContentSourceType::InlineText;
|
||||
else if (ContentTypeString == TEXT("TextFile")) Source.ContentType = EContentSourceType::TextFile;
|
||||
else if (ContentTypeString == TEXT("InlineText")) Source.ContentType = EContentSourceType::InlineText;
|
||||
else Source.ContentType = EContentSourceType::InlineText;
|
||||
|
||||
return Source;
|
||||
|
||||
@@ -4,9 +4,12 @@
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "GameFramework/Actor.h"
|
||||
#include "GestureData.h"
|
||||
#include "GestureAction.generated.h"
|
||||
|
||||
class AHandActor;
|
||||
class ATableIconActor;
|
||||
class AHandManager;
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API AGestureAction : public AActor
|
||||
@@ -22,8 +25,21 @@ protected:
|
||||
virtual void BeginPlay() override;
|
||||
|
||||
public:
|
||||
// Called every frame
|
||||
virtual void Tick(float DeltaTime) override;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
float UEUnitsPerUnit = 100.f;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly)
|
||||
bool bRequiresSecondHand = false;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly)
|
||||
bool bExtendable = false;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly)
|
||||
FGestureBinding Binding;
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
TArray<ATableIconActor*> Icons;
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
TMap<FName, FVector> Anchors;
|
||||
@@ -34,15 +50,29 @@ public:
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
AHandActor* HandB;
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
AGestureAction* LinkedActions;
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Stuff")
|
||||
AHandManager* HandManager;
|
||||
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void InitAction(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit);
|
||||
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void UpdateAction();
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void AbortAction();
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
void AddExtension(AHandActor* OtherHand);
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void AddExtension(AGestureAction* OtherAction);
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void AddOtherHand(AHandActor* OtherHand);
|
||||
|
||||
UFUNCTION(BlueprintCallable, BlueprintNativeEvent)
|
||||
void EndGesture();
|
||||
|
||||
void DestroyIcons();
|
||||
};
|
||||
|
||||
@@ -19,7 +19,7 @@ enum class EHandedness : uint8
|
||||
};
|
||||
|
||||
UENUM(BlueprintType)
|
||||
enum class EGesturePhase : uint8
|
||||
enum class EHandGesturePhase : uint8
|
||||
{
|
||||
None,
|
||||
Candidate,
|
||||
@@ -54,10 +54,10 @@ struct FExtendedGestureBinding
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
FString GestureName;
|
||||
FName GestureName;
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
FString GestureAction;
|
||||
FName GestureAction;
|
||||
};
|
||||
|
||||
// What gets parsed from the JSON
|
||||
@@ -67,15 +67,15 @@ struct FGestureBinding
|
||||
GENERATED_BODY()
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
FString ActionName;
|
||||
FName ActionName;
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
FString LeftGesture;
|
||||
FName LeftGesture;
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
FString RightGesture;
|
||||
FName RightGesture;
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
TArray<FExtendedGestureBinding> ExentdedGestures;
|
||||
TArray<FExtendedGestureBinding> ExtendedGestures;
|
||||
|
||||
};
|
||||
@@ -31,8 +31,7 @@ class TABLE_API UGestureDefinition : public UObject
|
||||
public:
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
FString GestureName;
|
||||
|
||||
FName GestureName;
|
||||
|
||||
// This checks the gesture and returns an aproximate scor of how well the Gesture was exectuded, needs to be overwritten in child classes.
|
||||
UFUNCTION(BlueprintCallable)
|
||||
@@ -50,13 +49,96 @@ protected:
|
||||
float GetFingersAverage(const AHandActor* Hand, const TArray<EHandFinger>& Fingers, bool bExtended) const;
|
||||
FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const;
|
||||
const FVector& GetPoint(const AHandActor* Hand, int32 Index) const;
|
||||
bool IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold = 0.7f) const;
|
||||
};
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API UGesture_PinchPoint : public UGestureDefinition
|
||||
class TABLE_API UGesture_PinchIndex : public UGestureDefinition
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual float Evaluate(AHandActor* Hand) const override;
|
||||
virtual void PostInitProperties() override
|
||||
{
|
||||
Super::PostInitProperties();
|
||||
|
||||
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||
{
|
||||
GestureName = TEXT("PinchIndex");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API UGesture_PinchMiddleIndex : public UGestureDefinition
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual float Evaluate(AHandActor* Hand) const override;
|
||||
virtual void PostInitProperties() override
|
||||
{
|
||||
Super::PostInitProperties();
|
||||
|
||||
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||
{
|
||||
GestureName = FName("PinchMiddle");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API UGesture_PointIndex : public UGestureDefinition
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual float Evaluate(AHandActor* Hand) const override;
|
||||
virtual void PostInitProperties() override
|
||||
{
|
||||
Super::PostInitProperties();
|
||||
|
||||
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||
{
|
||||
GestureName = FName("PointIndex");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API UGesture_PointMiddle : public UGestureDefinition
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual float Evaluate(AHandActor* Hand) const override;
|
||||
virtual void PostInitProperties() override
|
||||
{
|
||||
Super::PostInitProperties();
|
||||
|
||||
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||
{
|
||||
GestureName = TEXT("PointMiddle");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API UGesture_PointIndexMiddle : public UGestureDefinition
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
virtual float Evaluate(AHandActor* Hand) const override;
|
||||
virtual void PostInitProperties() override
|
||||
{
|
||||
Super::PostInitProperties();
|
||||
|
||||
if (!HasAnyFlags(RF_ClassDefaultObject))
|
||||
{
|
||||
GestureName = TEXT("PointIndexMiddle");
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -7,7 +7,7 @@
|
||||
#include "GestureData.h"
|
||||
#include "GestureManagerComponent.generated.h"
|
||||
|
||||
|
||||
class AHandManager;
|
||||
|
||||
UCLASS(ClassGroup = "Gesture", BlueprintType, Blueprintable, ClassGroup=(Custom), meta=(BlueprintSpawnableComponent) )
|
||||
class TABLE_API UGestureManagerComponent : public UActorComponent
|
||||
@@ -24,8 +24,11 @@ protected:
|
||||
|
||||
public:
|
||||
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Scale")
|
||||
float UEUnitsPerUnit = 100.f; // set via calibration rectangle
|
||||
|
||||
UPROPERTY(BlueprintReadOnly)
|
||||
class AHandManager* HandManager;
|
||||
AHandManager* HandManager;
|
||||
|
||||
UFUNCTION(BlueprintCallable, Category = "Gesture")
|
||||
void EvaluateHands();
|
||||
@@ -57,13 +60,13 @@ public:
|
||||
|
||||
// Populated on BeginPlay from the tables
|
||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
||||
TMap<FString, UGestureDefinition*> DefinitionMap;
|
||||
TMap<FName, UGestureDefinition*> DefinitionMap;
|
||||
|
||||
UPROPERTY(BlueprintReadOnly, Category = "Gesture")
|
||||
TMap<FString, TSubclassOf<AGestureAction>> ActionClassMap;
|
||||
TMap<FName, TSubclassOf<AGestureAction>> ActionClassMap;
|
||||
|
||||
private:
|
||||
// GestureManager.h — add these
|
||||
//
|
||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
||||
UDataTable* GestureDefinitionTable;
|
||||
|
||||
@@ -73,12 +76,13 @@ private:
|
||||
|
||||
void CheckGestureDefinitions(TArray<AHandActor*>& Hands);
|
||||
void TriggerAction(AHandActor* Hand);
|
||||
void UpdateOngoingAction();
|
||||
void EndGesture();
|
||||
void UpdateOngoingActions();
|
||||
|
||||
void SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B);
|
||||
void ExtendGestureAction(FString ActionName, AHandActor* Hand);
|
||||
void SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand);
|
||||
AGestureAction* SpawnGestureAction(FName ActionName, AHandActor* Hand);
|
||||
bool TryExtendGestureAction(AHandActor* Hand);
|
||||
|
||||
bool TryJoinExistingAction(AHandActor* Hand);
|
||||
bool TryCreateExtension(AHandActor* Hand);
|
||||
|
||||
static constexpr int32 CandidateThreshold = 8;
|
||||
static constexpr int32 FailThreshold = 5;
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
class AHandManager;
|
||||
class AFingerTip;
|
||||
class AGestureAction;
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API AHandActor : public AActor
|
||||
@@ -47,10 +48,13 @@ public:
|
||||
// Gestures
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
EGesturePhase Phase = EGesturePhase::None;
|
||||
EHandGesturePhase Phase = EHandGesturePhase::None;
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
FString ActiveGestureName;
|
||||
FName ActiveGestureName;
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
AGestureAction* ActiveAction;
|
||||
|
||||
UPROPERTY(BlueprintReadWrite)
|
||||
TMap<FString, FVector> Anchors;
|
||||
@@ -67,15 +71,7 @@ public:
|
||||
bool bUsedThisTick = false;
|
||||
|
||||
bool IsPaired() const { return PairedHand != nullptr; }
|
||||
|
||||
void ResetGesture()
|
||||
{
|
||||
Phase = EGesturePhase::None;
|
||||
ActiveGestureName = "";
|
||||
CandidateFrames = 0;
|
||||
FailFrames = 0;
|
||||
}
|
||||
|
||||
void ResetGesture();
|
||||
|
||||
protected:
|
||||
// Called when the game starts or when spawned
|
||||
|
||||
@@ -62,7 +62,7 @@ public:
|
||||
FOnHandRemoved OnHandRemoved;
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
TArray<AHandActor*> GetHandList();
|
||||
TArray<AHandActor*>& GetHandList();
|
||||
|
||||
//Gestures
|
||||
|
||||
@@ -72,12 +72,13 @@ public:
|
||||
const TArray<AHandActor*>& GetHandList() const { return HandList; }
|
||||
|
||||
// GestureManager.h — add these
|
||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture")
|
||||
UDataTable* GestureDefinitionTable;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, Category = "Gesture")
|
||||
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Gesture")
|
||||
UDataTable* GestureActionTable;
|
||||
|
||||
|
||||
private:
|
||||
void RemoveDeadHands();
|
||||
void UpdateHands();
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "GestureAction.h"
|
||||
#include "MyGestureActions.generated.h"
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
UCLASS()
|
||||
class TABLE_API AMyGestureActions : public AGestureAction
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
};
|
||||
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API ARulerAction : public AGestureAction
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
void UpdateAction_Implementation() override;
|
||||
void AbortAction_Implementation() override;
|
||||
void AddExtension_Implementation(AGestureAction* OtherAction) override;
|
||||
void EndGesture_Implementation() override;
|
||||
void InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit) override;
|
||||
|
||||
private:
|
||||
float GetSnappedInches(float RawDistance) const;
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "TableIconActor.h"
|
||||
#include "ProceduralMeshComponent.h"
|
||||
#include "Components/WidgetComponent.h"
|
||||
#include "Components/WidgetInteractionComponent.h"
|
||||
#include "MyTableIconActors.generated.h"
|
||||
|
||||
class UWidgetComponent;
|
||||
class UWidgetInteractionComponent;
|
||||
/**
|
||||
*
|
||||
*/
|
||||
UCLASS()
|
||||
class TABLE_API AMyTableIconActors : public ATableIconActor
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
};
|
||||
|
||||
|
||||
UCLASS(Blueprintable)
|
||||
class TABLE_API ATableIconLineActor : public ATableIconActor
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
ATableIconLineActor();
|
||||
|
||||
UFUNCTION(BlueprintCallable)
|
||||
void SetEndpoints(const FVector& Start, const FVector& End);
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
float LineThickness = 1.f;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
FLinearColor LineColor = FLinearColor::White;
|
||||
|
||||
// Tick markers along the line — e.g. every inch
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
bool bShowTickMarks = true;
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
float TickInterval = 1.f; // in real-world units
|
||||
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
float UEUnitsPerUnit = 100.f;
|
||||
|
||||
private:
|
||||
UPROPERTY()
|
||||
UProceduralMeshComponent* MeshComponent;
|
||||
|
||||
void BuildLineMesh(const FVector& Start, const FVector& End);
|
||||
void BuildTickMesh(const FVector& Start, const FVector& End);
|
||||
|
||||
FVector LastStart;
|
||||
FVector LastEnd;
|
||||
};
|
||||
|
||||
|
||||
|
||||
UCLASS(Blueprintable)
|
||||
class TABLE_API ATableIconWidgetActor : public ATableIconActor
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
ATableIconWidgetActor();
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly)
|
||||
UWidgetComponent* WidgetComponent;
|
||||
|
||||
UPROPERTY(VisibleAnywhere, BlueprintReadOnly)
|
||||
UWidgetInteractionComponent* WidgetInteraction;
|
||||
|
||||
// Face toward a target each tick — useful for always facing the camera
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
bool bFaceCamera = true;
|
||||
|
||||
// Set widget content — callable from GestureAction
|
||||
UFUNCTION(BlueprintCallable)
|
||||
void SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass);
|
||||
|
||||
// Get the underlying widget cast to a specific type in BP
|
||||
UFUNCTION(BlueprintCallable, BlueprintPure)
|
||||
UUserWidget* GetWidget() const;
|
||||
|
||||
protected:
|
||||
virtual void Tick(float DeltaTime) override;
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
// Fill out your copyright notice in the Description page of Project Settings.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "CoreMinimal.h"
|
||||
#include "GameFramework/Actor.h"
|
||||
#include "TableIconActor.generated.h"
|
||||
|
||||
UCLASS()
|
||||
class TABLE_API ATableIconActor : public AActor
|
||||
{
|
||||
GENERATED_BODY()
|
||||
|
||||
public:
|
||||
// Sets default values for this actor's properties
|
||||
ATableIconActor();
|
||||
|
||||
protected:
|
||||
// Called when the game starts or when spawned
|
||||
virtual void BeginPlay() override;
|
||||
|
||||
public:
|
||||
// If true, GestureAction will not destroy this on EndGesture
|
||||
UPROPERTY(EditDefaultsOnly, BlueprintReadWrite)
|
||||
bool bPersistOnActionEnd = false;
|
||||
|
||||
// Called by GestureAction on end — BP overrides this for fade/lock behavior
|
||||
UFUNCTION(BlueprintNativeEvent)
|
||||
void OnActionEnded();
|
||||
|
||||
// Called by GestureAction every tick
|
||||
UFUNCTION(BlueprintNativeEvent)
|
||||
void OnActionUpdated();
|
||||
|
||||
|
||||
};
|
||||
@@ -20,7 +20,8 @@ public class Table : ModuleRules
|
||||
"UMG" ,
|
||||
"Json",
|
||||
"JsonUtilities",
|
||||
"ImageWrapper"
|
||||
"ImageWrapper",
|
||||
"ProceduralMeshComponent"
|
||||
});
|
||||
|
||||
PrivateDependencyModuleNames.AddRange(new string[] { });
|
||||
|
||||
@@ -8,8 +8,8 @@ public class TableEditorTarget : TargetRules
|
||||
public TableEditorTarget( TargetInfo Target) : base(Target)
|
||||
{
|
||||
Type = TargetType.Editor;
|
||||
DefaultBuildSettings = BuildSettingsVersion.V6;
|
||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_7;
|
||||
DefaultBuildSettings = BuildSettingsVersion.V7;
|
||||
IncludeOrderVersion = EngineIncludeOrderVersion.Unreal5_8;
|
||||
ExtraModuleNames.Add("Table");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
<Solution>
|
||||
<Configurations>
|
||||
<BuildType Name="DebugGame" />
|
||||
<BuildType Name="DebugGame Editor" />
|
||||
<BuildType Name="Development" />
|
||||
<BuildType Name="Development Editor" />
|
||||
<BuildType Name="Shipping" />
|
||||
<Platform Name="Win64" />
|
||||
<Platform Name="Win64-arm64" />
|
||||
<Platform Name="Win64-arm64ec" />
|
||||
</Configurations>
|
||||
<Folder Name="/Engine/">
|
||||
<Project Path="Intermediate/ProjectFiles/UE5.vcxproj" Id="99002b2b-f5da-337f-af6f-e86d1d78a0d0">
|
||||
<BuildType Project="BuiltWithUnrealBuildTool" />
|
||||
<Platform Project="Win64" />
|
||||
<Build Project="false" />
|
||||
</Project>
|
||||
</Folder>
|
||||
<Folder Name="/Games/">
|
||||
<Project Path="Intermediate/ProjectFiles/Table.vcxproj" Id="111ae310-8395-3d64-9119-33389f569f2d">
|
||||
<BuildType Solution="DebugGame Editor|Win64" Project="DebugGame_Editor" />
|
||||
<BuildType Solution="DebugGame Editor|Win64-arm64" Project="Win64_arm64_DebugGame_Editor" />
|
||||
<BuildType Solution="DebugGame Editor|Win64-arm64ec" Project="Win64_arm64ec_DebugGame_Editor" />
|
||||
<BuildType Solution="DebugGame|Win64-arm64" Project="Win64_arm64_DebugGame" />
|
||||
<BuildType Solution="DebugGame|Win64-arm64ec" Project="Win64_arm64ec_DebugGame" />
|
||||
<BuildType Solution="Development Editor|Win64" Project="Development_Editor" />
|
||||
<BuildType Solution="Development Editor|Win64-arm64" Project="Win64_arm64_Development_Editor" />
|
||||
<BuildType Solution="Development Editor|Win64-arm64ec" Project="Win64_arm64ec_Development_Editor" />
|
||||
<BuildType Solution="Development|Win64-arm64" Project="Win64_arm64_Development" />
|
||||
<BuildType Solution="Development|Win64-arm64ec" Project="Win64_arm64ec_Development" />
|
||||
<BuildType Solution="Shipping|Win64-arm64" Project="Win64_arm64_Shipping" />
|
||||
<BuildType Solution="Shipping|Win64-arm64ec" Project="Win64_arm64ec_Shipping" />
|
||||
<Platform Project="x64" />
|
||||
</Project>
|
||||
</Folder>
|
||||
<Folder Name="/Visualizers/">
|
||||
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natjmc" />
|
||||
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natstepfilter" />
|
||||
<File Path="G:/Unreal/UE_5.8/Engine/Extras/VisualStudioDebugging/Unreal.natvis" />
|
||||
</Folder>
|
||||
<Properties Name="ddbf523f-7eb6-4887-bd51-85a714ff87eb">
|
||||
<Property Name="AvailablePlatforms" Value="Win64" />
|
||||
</Properties>
|
||||
</Solution>
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"FileVersion": 3,
|
||||
"EngineAssociation": "5.7",
|
||||
"EngineAssociation": "5.8",
|
||||
"Category": "",
|
||||
"Description": "",
|
||||
"Modules": [
|
||||
@@ -22,7 +22,7 @@
|
||||
},
|
||||
{
|
||||
"Name": "VisualStudioTools",
|
||||
"Enabled": true,
|
||||
"Enabled": false,
|
||||
"SupportedTargetPlatforms": [
|
||||
"Win64"
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user