Added Gesture Actions, Gesture Definitions, TableIcons, ...

Table/Source/Table/Private/GestureAction.cpp

@@ -3,44 +3,62 @@
 
 #include "GestureAction.h"
 #include "HandActor.h"
+#include "TableIconActor.h"
 
 // Sets default values
 AGestureAction::AGestureAction()
 {
  	// Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
-	PrimaryActorTick.bCanEverTick = true;
-
+	PrimaryActorTick.bCanEverTick = false;
 }
 
 // Called when the game starts or when spawned
 void AGestureAction::BeginPlay()
 {
 	Super::BeginPlay();
-	
 }
 
-// Called every frame
-void AGestureAction::Tick(float DeltaTime)
+void AGestureAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
 {
-	Super::Tick(DeltaTime);
-
+	HandA = Hand;
+	bRequiresSecondHand = Binding.LeftGesture != "" && Binding.RightGesture != "";
+	bExtendable = Binding.ExtendedGestures.IsEmpty();
+	Binding = NewBinding;
+	HandManager = Hand->HandManager;
+	UEUnitsPerUnit = NewUEUnitsPerUnit;
 }
 
-void AGestureAction::UpdateAction()
+void AGestureAction::UpdateAction_Implementation()
 {
 	Destroy();
 }
 
-void AGestureAction::AbortAction()
+void AGestureAction::AbortAction_Implementation()
 {
+	for (ATableIconActor* Icon : Icons)
+		Icon->bPersistOnActionEnd = false;
+	EndGesture();
 }
 
-void AGestureAction::AddExtension(AHandActor* OtherHand)
+void AGestureAction::AddExtension_Implementation(AGestureAction* OtherAction)
+{
+	LinkedActions = OtherAction;
+}
+
+void AGestureAction::AddOtherHand_Implementation(AHandActor* OtherHand)
 {
 	HandB = OtherHand;
 }
 
-void AGestureAction::EndGesture()
+void AGestureAction::EndGesture_Implementation()
 {
+	DestroyIcons();
 	Destroy();
+}
+
+void AGestureAction::DestroyIcons()
+{
+	for (ATableIconActor* Icon : Icons)
+		if (IsValid(Icon) ) Icon->OnActionEnded();
+	Icons.Empty();
 }

Table/Source/Table/Private/GestureDefinition.cpp

@@ -40,6 +40,11 @@ const FVector& UGestureDefinition::GetPoint(const AHandActor* Hand, int32 Index)
     return Hand->Points[Index];
 }
 
+bool UGestureDefinition::IsFingerExtended(const AHandActor* Hand, EHandFinger Finger, float Threshold) const
+{
+    return GetFingerExtendedScore(Hand, Finger) > Threshold;
+}
+
 float UGestureDefinition::GetHandScale(const AHandActor* Hand) const
 {
     const TArray<FVector>& P = Hand->Points;
@@ -53,30 +58,44 @@ float UGestureDefinition::GetNormalizedDistance(const AHandActor* Hand, int32 A,
 }
 float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const
 {
-    float Dist = GetNormalizedDistance(
-        Hand,
-        GetFingerTipIndex(A),
-        GetFingerTipIndex(B)
-    );
+    const FVector& P1 = Hand->Points[GetFingerTipIndex(A)];
+    const FVector& P2 = Hand->Points[GetFingerTipIndex(B)];
+
+    float Dist = FVector::Dist(P1, P2);
+
+    float HandScale =  FVector::Dist( Hand->Points[0], Hand->Points[9] );
+
+    float Normalized = Dist / HandScale;
+
+    //UE_LOG(LogTemp, Warning, TEXT("Dist: %.3f | HandScale: %.3f | Norm: %.3f"), Dist, HandScale, Normalized);
 
     const float Threshold = 0.25f;
 
-    return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f);
+    return 1.f - FMath::Clamp( Normalized / Threshold, 0.f, 1.f );
 }
 
 float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const
 {
     const TArray<FVector>& P = Hand->Points;
 
+    if (Finger == EHandFinger::Thumb)
+    {
+        // Thumb: compare tip->base against index base->wrist as reference direction
+        const FVector& Base = P[2];
+        const FVector& Tip = P[4];
+        const FVector& Ref = P[5];  // index base as stable reference
+        float Dot = FVector::DotProduct(
+            (Tip - Base).GetSafeNormal(),
+            (Ref - P[0]).GetSafeNormal());
+        return FMath::Clamp(Dot, 0.f, 1.f);
+    }
+
     const FVector& Wrist = P[0];
     const FVector& Base = P[GetFingerBaseIndex(Finger)];
     const FVector& Tip = P[GetFingerTipIndex(Finger)];
-
     float Dot = FVector::DotProduct(
         (Tip - Base).GetSafeNormal(),
-        (Base - Wrist).GetSafeNormal()
-    );
-
+        (Base - Wrist).GetSafeNormal());
     return FMath::Clamp(Dot, 0.f, 1.f);
 }
 
@@ -91,9 +110,7 @@ float UGestureDefinition::GetFingersAverage(const AHandActor* Hand, const TArray
 
     for (EHandFinger Finger : Fingers)
     {
-        Sum += bExtended
-            ? GetFingerExtendedScore(Hand, Finger)
-            : GetFingerCurledScore(Hand, Finger);
+        Sum += bExtended ? GetFingerExtendedScore(Hand, Finger) : GetFingerCurledScore(Hand, Finger);
     }
 
     return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f;
@@ -107,19 +124,57 @@ FVector UGestureDefinition::GetFingerDirection(const AHandActor* Hand, EHandFing
     return (Tip - Base).GetSafeNormal();
 }
 
-float UGesture_PinchPoint::Evaluate(AHandActor* Hand) const
+float UGesture_PinchIndex::Evaluate(AHandActor* Hand) const
 {
     float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
     float IndexExtended = GetFingerExtendedScore(Hand, EHandFinger::Index);
 
-    float OthersCurled = GetFingersAverage(
-        Hand,
-        { EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky },
-        false
-    );
+    float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Middle, EHandFinger::Ring, EHandFinger::Pinky }, false );
 
-    return
-        Pinch * 0.5f +
-        IndexExtended * 0.3f +
-        OthersCurled * 0.2f;
+    return Pinch * 0.5f + IndexExtended * 0.3f + OthersCurled * 0.2f;
+}
+
+float UGesture_PinchMiddleIndex::Evaluate(AHandActor* Hand) const
+{
+    float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Middle);
+    Pinch += GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index);
+
+    float MiddleExtended = GetFingerExtendedScore(Hand, EHandFinger::Middle);
+    float OthersCurled = GetFingersAverage( Hand, { EHandFinger::Ring, EHandFinger::Pinky }, false );
+
+    return Pinch * 0.5f + MiddleExtended * 0.3f + OthersCurled * 0.2f;
+
+}
+
+float UGesture_PointIndex::Evaluate(AHandActor* Hand) const
+{
+    float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
+    float Middle = GetFingerCurledScore(Hand, EHandFinger::Middle);
+    float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
+    float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
+    float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
+     
+    return ( Index + Middle + Ring + Pinky + Thumb ) / 5.f;
+}
+
+float UGesture_PointMiddle::Evaluate(AHandActor* Hand) const
+{
+    float Index = GetFingerCurledScore(Hand, EHandFinger::Index);
+    float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
+    float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
+    float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
+    float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
+
+    return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
+}
+
+float UGesture_PointIndexMiddle::Evaluate(AHandActor* Hand) const
+{
+    float Index = GetFingerExtendedScore(Hand, EHandFinger::Index);
+    float Middle = GetFingerExtendedScore(Hand, EHandFinger::Middle);
+    float Ring = GetFingerCurledScore(Hand, EHandFinger::Ring);
+    float Pinky = GetFingerCurledScore(Hand, EHandFinger::Pinky);
+    float Thumb = GetFingerCurledScore(Hand, EHandFinger::Thumb);
+
+    return (Index + Middle + Ring + Pinky + Thumb) / 5.f;
 }

Table/Source/Table/Private/GestureManagerComponent.cpp

@@ -14,7 +14,7 @@ UGestureManagerComponent::UGestureManagerComponent()
 	// off to improve performance if you don't need them.
 	PrimaryComponentTick.bCanEverTick = false;
 
-	HandManager = Cast<AHandManager>(GetOwner());
+	
 }
 
 
@@ -23,6 +23,8 @@ void UGestureManagerComponent::BeginPlay()
 {
 	Super::BeginPlay();
 
+	HandManager = Cast<AHandManager>(GetOwner());
+
 	HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded);
 	HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved);
 
@@ -42,17 +44,16 @@ void UGestureManagerComponent::EvaluateHands()
 		if (Hand->bUsedThisTick) continue;
 		switch (Hand->Phase)
 		{
-		case EGesturePhase::Started:
+		case EHandGesturePhase::Started:
 			TriggerAction(Hand);
 			break;
-		case EGesturePhase::Ended:
+		case EHandGesturePhase::Ended:
 			Hand->ResetGesture();
-			EndGesture();
 			break;
 		}
 	}
 
-	UpdateOngoingAction();
+	UpdateOngoingActions();
 }
 
 void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
@@ -62,14 +63,16 @@ void UGestureManagerComponent::OnHandAdded(AHandActor* Hand)
 
 void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand)
 {
-	Hand->PairedHand->PairedHand = nullptr;
-	Hand->ResetGesture();
+	if (!Hand->ActiveAction) return;
+	if (!ActiveGestureActions.Contains(Hand->ActiveAction)) return;
+	Hand->ActiveAction->Destroy();
+	ActiveGestureActions.Remove(Hand->ActiveAction);
 }
 
 void UGestureManagerComponent::LoadBindings(const FString& FilePath)
 {
-	FString JsonString;
-
+	FString JsonString = "";
+	FString Temp = "";
 	if (!FFileHelper::LoadFileToString(JsonString, *FilePath))
 	{
 		return;
@@ -92,9 +95,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
 
 			FGestureBinding Binding;
 
-			Obj->TryGetStringField(TEXT("action"), Binding.ActionName);
-			Obj->TryGetStringField(TEXT("left"), Binding.LeftGesture);
-			Obj->TryGetStringField(TEXT("right"), Binding.RightGesture);
+			Obj->TryGetStringField(TEXT("action"), Temp);
+			Binding.ActionName = FName(Temp);
+			Obj->TryGetStringField(TEXT("left"), Temp);
+			Binding.LeftGesture = FName(Temp);
+			Obj->TryGetStringField(TEXT("right"), Temp);
+			Binding.RightGesture = FName(Temp);
 
 			const TArray<TSharedPtr<FJsonValue>>* ExtArray;
 			if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray))
@@ -106,10 +112,12 @@ void UGestureManagerComponent::LoadBindings(const FString& FilePath)
 
 					FExtendedGestureBinding Ext;
 
-					ExtObj->TryGetStringField(TEXT("gesture"), Ext.GestureName);
-					ExtObj->TryGetStringField(TEXT("action"), Ext.GestureAction);
+					ExtObj->TryGetStringField(TEXT("gesture"), Temp);
+					Ext.GestureName = FName(Temp);
+					ExtObj->TryGetStringField(TEXT("action"), Temp);
+					Ext.GestureAction = FName(Temp);
 
-					Binding.ExentdedGestures.Add(Ext);
+					Binding.ExtendedGestures.Add(Ext);
 				}
 			}
 
@@ -143,6 +151,7 @@ void UGestureManagerComponent::BuildDefinitionMap()
 		UGestureDefinition* Def = NewObject<UGestureDefinition>(this, Row->DefinitionClass);
 
 		DefinitionMap.Add(Def->GestureName, Def);
+		GestureDefinitions.Add(Def);
 	}
 }
 
@@ -159,176 +168,171 @@ void UGestureManagerComponent::BuildActionMap()
 	{
 		if (!Row || !Row->ActionClass) continue;
 
-		ActionClassMap.Add(
-			Row->ActionClass->GetName(),
-			Row->ActionClass
-
-		);
+		ActionClassMap.Add( FName(Row->ActionClass->GetName()), Row->ActionClass );
 	}
 }
 
 void UGestureManagerComponent::CheckGestureDefinitions(TArray<AHandActor*>& Hands)
 {
-	const float MinScoreThreshold = 0.5f;
+	//const float MinScoreThreshold = 0.7f;
+
 
 	for (AHandActor* Hand : Hands) {
+		float Threshold = (Hand->Phase == EHandGesturePhase::Ongoing || Hand->Phase == EHandGesturePhase::Started) ? 0.60f : 0.75f;
+
 		float BestScore = 0.f;
-		FString BestGesture = "";
+		FName BestGesture = "";
 
 		for (UGestureDefinition* Gesture : GestureDefinitions) {
 
-			float Score = Gesture->Evaluate(Hand);
+			float Score = Gesture->Evaluate(Hand); // Getting the Score of the gestures to check
 
-			if (Score > BestScore)
+			//UE_LOG(LogTemp, Warning, TEXT("%s -> %s : %.3f"), *Hand->GetName(), *Gesture->GestureName, Score);
+
+			if (Score > BestScore) // Getting the gesture with the highest Score
 			{
 				BestScore = Score;
 				BestGesture = Gesture->GestureName;
 			}
 		}
 
-		if (BestScore < MinScoreThreshold)
+		// Check if the gesture with the highest score is a valid Gesture
+		if (BestScore < Threshold)
 		{
-			if (Hand->Phase != EGesturePhase::None)
+			if (Hand->Phase != EHandGesturePhase::None)
 			{
 				Hand->FailFrames++;
-
+				UE_LOG(LogTemp, Warning,
+					TEXT("[%s] Below threshold (%s) Score=%.3f Fail=%d/%d"),
+					*Hand->GetName(),
+					*Hand->ActiveGestureName.ToString(),
+					BestScore,
+					Hand->FailFrames,
+					FailThreshold);
+				//if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
 				if (Hand->FailFrames >= FailThreshold)
 				{
-					Hand->Phase = EGesturePhase::Ended;
+					UE_LOG(LogTemp, Warning,
+						TEXT("[%s] -> ENDED (%s)"),
+						*Hand->GetName(),
+						*Hand->ActiveGestureName.ToString());
+
+					Hand->Phase = EHandGesturePhase::Ended;
 				}
 			}
-
 			continue;
 		}
 
+		// Handle valid Gesture
 		switch (Hand->Phase)
 		{
-		case EGesturePhase::None:
+		case EHandGesturePhase::None: // Starting a Candidate
 			Hand->ActiveGestureName = BestGesture;
-			Hand->Phase = EGesturePhase::Candidate;
+			Hand->Phase = EHandGesturePhase::Candidate;
 			Hand->CandidateFrames = 1;
 			Hand->FailFrames = 0;
+			UE_LOG(LogTemp, Warning,
+				TEXT("[%s] NONE -> CANDIDATE (%s)"),
+				*Hand->GetName(),
+				*BestGesture.ToString());
 
 			break;
-		case EGesturePhase::Candidate:
-			if (Hand->ActiveGestureName != BestGesture)
+		case EHandGesturePhase::Candidate: // Checking if the Candidate stays valid or another gesture is made
+			if (Hand->ActiveGestureName == BestGesture)
 			{ 
-				Hand->FailFrames++;
-				break; 
+				Hand->CandidateFrames++;
+				Hand->FailFrames = 0;
+
+				UE_LOG(LogTemp, Warning,
+					TEXT("[%s] Candidate %s (%d/%d)"),
+					*Hand->GetName(),
+					*BestGesture.ToString(),
+					Hand->CandidateFrames,
+					CandidateThreshold);
+
+				//if (Hand->CandidateFrames >= CandidateThreshold) Hand->Phase = EHandGesturePhase::Started;
+				if (Hand->CandidateFrames >= CandidateThreshold)
+				{
+					Hand->Phase = EHandGesturePhase::Started;
+
+					UE_LOG(LogTemp, Warning,
+						TEXT("[%s] CANDIDATE -> STARTED (%s)"),
+						*Hand->GetName(),
+						*BestGesture.ToString());
+				}
+				break;
 			}
+			UE_LOG(LogTemp, Warning,
+				TEXT("[%s] Candidate switched %s -> %s"),
+				*Hand->GetName(),
+				*Hand->ActiveGestureName.ToString(),
+				*BestGesture.ToString());
 
-			Hand->CandidateFrames++;
+			Hand->ActiveGestureName = BestGesture;
+			Hand->CandidateFrames = 1;
 			Hand->FailFrames = 0;
-
-			if (Hand->CandidateFrames >= CandidateThreshold)
-				Hand->Phase = EGesturePhase::Started;
-			
 			break;
-		case EGesturePhase::Ongoing:
+		case EHandGesturePhase::Ongoing: // Checking if the gesture is still ongoing
+			//if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
+			//else Hand->FailFrames = 0;
 			if (Hand->ActiveGestureName != BestGesture)
 			{
 				Hand->FailFrames++;
 
-				if (Hand->FailFrames >= FailThreshold)
-				{
-					Hand->Phase = EGesturePhase::Ended;
-				}
+				UE_LOG(LogTemp, Warning,
+					TEXT("[%s] Ongoing fail %s (%d/%d)"),
+					*Hand->GetName(),
+					*Hand->ActiveGestureName.ToString(),
+					Hand->FailFrames,
+					FailThreshold);
 			}
 			else
 			{
-				Hand->FailFrames = 0;
+				if (Hand->FailFrames > 0)
+				{
+					UE_LOG(LogTemp, Warning,
+						TEXT("[%s] Ongoing recovered %s"),
+						*Hand->GetName(),
+						*Hand->ActiveGestureName.ToString());
+				}
 			}
 			break;
+		//case EHandGesturePhase::Started:
+		//	break;
+		default: // If the gesture would be stuck somewhere
+			if (Hand->ActiveGestureName != BestGesture) Hand->FailFrames++;
+
+			if (Hand->FailFrames >= FailThreshold) Hand->ResetGesture(); // This optimaly shouldnt be called here
+			break;
 		}
 
-		
-
-		if (Hand->FailFrames >= FailThreshold)
-			Hand->Phase = EGesturePhase::Ended;
+		// Checking if a gesture has ended
+		if (Hand->FailFrames >= FailThreshold) Hand->Phase = EHandGesturePhase::Ended;
 	}
 }
 
 void UGestureManagerComponent::TriggerAction(AHandActor* Hand)
 {
-	if (!Hand->bUsedThisTick) return;
-
-	if (Hand->IsPaired()) 
-	{
-		AHandActor* Other = Hand->PairedHand;
-
-		for (const FGestureBinding& Binding : GestureBindings)
-		{
-			const bool bLeftMatch = (Hand->Handedness == EHandedness::Left && Hand->ActiveGestureName == Binding.LeftGesture &&
-				Other->ActiveGestureName == Binding.RightGesture);
-
-			const bool bRightMatch = (Hand->Handedness == EHandedness::Right && Hand->ActiveGestureName == Binding.RightGesture &&
-				Other->ActiveGestureName == Binding.LeftGesture);
-
-			if (!(bLeftMatch || bRightMatch))
-				continue;
-
-			// both must be ready
-			if (Other->Phase != EGesturePhase::Started)
-				continue;
-
-			// mark used
-			Hand->bUsedThisTick = true;
-			Other->bUsedThisTick = true;
-
-			Hand->Phase = EGesturePhase::Ongoing;
-			Other->Phase = EGesturePhase::Ongoing;
-
-			// spawn action (centralized factory recommended)
-			SpawnGestureAction(Binding.ActionName, Hand, Other);
-
-			return;
-		}
-	}
+	if (Hand->bUsedThisTick) return;
 	
+	if (TryExtendGestureAction(Hand)) return;
+
 	for (const FGestureBinding& Binding : GestureBindings)
 	{
-		const bool bMatch =
-			(Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName) ||
-			(Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
+		bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
+					|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
 
-		if (!bMatch)
-			continue;
+		if (!bMatches) continue;
 
-		// if paired hand is actively running something, allow extension check later
-		if (Hand->IsPaired() && Hand->PairedHand && Hand->PairedHand->Phase == EGesturePhase::Ongoing)
-		{
-			for (const FExtendedGestureBinding& Ext : Binding.ExentdedGestures)
-			{
-				if (Ext.GestureName != Hand->ActiveGestureName)
-					continue;
+		AGestureAction* NewAction = SpawnGestureAction(Binding.ActionName, Hand);
+		NewAction->InitAction(Hand, Binding, UEUnitsPerUnit);
 
-				if (Ext.GestureAction == Binding.ActionName)
-				{
-					// extend current action
-					ExtendGestureAction(Binding.ActionName, Hand);
-					Hand->bUsedThisTick = true;
-					return;
-				}
-				else
-				{
-					// branch: spawn new action linked to existing one
-					SpawnGestureActionWithParent(Ext.GestureAction, Hand, Hand->PairedHand);
-					Hand->bUsedThisTick = true;
-					return;
-				}
-			}
-		}
-
-		// normal one-hand action
-		Hand->bUsedThisTick = true;
-		SpawnGestureAction(Binding.ActionName, Hand, nullptr);
 		return;
 	}
-	
 }
 
 
-void UGestureManagerComponent::UpdateOngoingAction()
+void UGestureManagerComponent::UpdateOngoingActions()
 {
 	for (AGestureAction* Action : ActiveGestureActions)
 	{
@@ -338,18 +342,94 @@ void UGestureManagerComponent::UpdateOngoingAction()
 	}
 }
 
-void UGestureManagerComponent::SpawnGestureAction(FString ActionName, AHandActor* A, AHandActor* B)
+AGestureAction* UGestureManagerComponent::SpawnGestureAction(FName ActionName, AHandActor* Hand)
 {
+
+	TSubclassOf<AGestureAction>* ActionClassPtr = ActionClassMap.Find(ActionName);
+
+	if (!ActionClassPtr || !Hand) return nullptr;
+
+	FActorSpawnParameters SpawnParams;
+	SpawnParams.Owner = GetOwner();
+	SpawnParams.SpawnCollisionHandlingOverride =
+		ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
+
+	AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
+			*ActionClassPtr,
+			FVector::ZeroVector,
+			FRotator::ZeroRotator,
+			SpawnParams);
+
+	if (!NewAction) return nullptr;
+
+
+	Hand->ActiveAction = NewAction;
+	Hand->bUsedThisTick = true;
+	Hand->Phase = EHandGesturePhase::Ongoing;
+
+	return NewAction;
 }
 
-void UGestureManagerComponent::ExtendGestureAction(FString ActionName, AHandActor* Hand)
+bool UGestureManagerComponent::TryExtendGestureAction(AHandActor* Hand)
 {
+	if (!Hand->IsPaired()  || Hand->PairedHand->Phase != EHandGesturePhase::Ongoing || !Hand->PairedHand->ActiveAction) return false;
+
+	if (TryJoinExistingAction(Hand)) return true;  
+	if (TryCreateExtension(Hand)) return true;
+	
+
+	return false;
 }
 
-void UGestureManagerComponent::SpawnGestureActionWithParent(FString ActionName, AHandActor* Hand, AHandActor* ParentHand)
+bool UGestureManagerComponent::TryJoinExistingAction(AHandActor* Hand)
 {
+	AHandActor* OtherHand = Hand->PairedHand;
+	AGestureAction* OtherAction = OtherHand->ActiveAction;
+
+	if (!OtherAction->bRequiresSecondHand) return false; // Check if its a Two Handed Action or an extension
+
+	FGestureBinding& Binding = OtherAction->Binding;
+
+	bool bMatches = (Hand->Handedness == EHandedness::Left && Binding.LeftGesture == Hand->ActiveGestureName)
+		|| (Hand->Handedness == EHandedness::Right && Binding.RightGesture == Hand->ActiveGestureName);
+
+	if (!bMatches) return false;
+
+	OtherAction->AddOtherHand(Hand);
+	OtherHand->bUsedThisTick = true;
+	Hand->bUsedThisTick = true;
+	Hand->Phase = EHandGesturePhase::Ongoing;
+	Hand->ActiveAction = OtherAction;
+
+	return true;
 }
 
+bool UGestureManagerComponent::TryCreateExtension(AHandActor* Hand)
+{
+	AHandActor* OtherHand = Hand->PairedHand;
+	AGestureAction* OtherAction = OtherHand->ActiveAction;
 
+	if (!OtherHand->ActiveAction->bExtendable) return false;
 
+	FName ActionName;
+	for (const FExtendedGestureBinding& Gesture : OtherAction->Binding.ExtendedGestures)
+	{
+		if (Gesture.GestureName == Hand->ActiveGestureName)
+		{
+			ActionName = Gesture.GestureAction;
+			break;
+		}
+	}
+	if (ActionName.IsNone()) return false;
+
+	AGestureAction* NewAction = SpawnGestureAction(ActionName, Hand);
+
+	NewAction->InitAction(Hand, OtherHand->ActiveAction->Binding, UEUnitsPerUnit);
+	OtherHand->bUsedThisTick = true;
+	Hand->bUsedThisTick = true;
+	OtherAction->AddExtension(NewAction);
+	NewAction->AddExtension(OtherHand->ActiveAction);
+
+	return true;
+}
 

Table/Source/Table/Private/HandActor.cpp

@@ -4,6 +4,7 @@
 #include "HandManager.h"
 #include "FingerTip.h"
 #include "DrawDebugHelpers.h"
+#include "GestureAction.h"
 
 
 static const int32 HandBones[][2] = {
@@ -31,14 +32,21 @@ AHandActor::AHandActor()
 
 }
 
+void AHandActor::ResetGesture()
+{
+	Phase = EHandGesturePhase::None;
+	ActiveGestureName = "";
+	CandidateFrames = 0;
+	FailFrames = 0;
+	if (IsValid(ActiveAction)) ActiveAction->EndGesture();
+	ActiveAction = nullptr;
+}
+
 // Called when the game starts or when spawned
 void AHandActor::BeginPlay()
 {
 	Super::BeginPlay();
 	GetWorld()->GetTimerManager().SetTimer(DeathTimer, this, &AHandActor::HandDeath, DeathTime, true);
-
-
-
 }
 
 // Called every frame
@@ -123,8 +131,6 @@ void AHandActor::UpdateHand(const FHandPacketHand& NewHand)
 	Confidence = NewHand.Confidence;
 	HandId = NewHand.HandId;
 
-	//UE_LOG(LogTemp, Error, TEXT("Update Hand"));
-
 	for (int32 n = 0; n < 5; n++)
 	{
 		FVector f = Points[FingerTipBones[n]];

Table/Source/Table/Private/HandManager.cpp

@@ -10,7 +10,7 @@ AHandManager::AHandManager()
  	// Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
 	PrimaryActorTick.bCanEverTick = true;
 	GestureManager = CreateDefaultSubobject<UGestureManagerComponent>(TEXT("GestureManager"));
-	GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
+	
 }
 
 // Called when the game starts or when spawned
@@ -18,6 +18,8 @@ void AHandManager::BeginPlay()
 {
 	Super::BeginPlay();
 
+	GestureManager->Initialize(GestureDefinitionTable, GestureActionTable);
+
 	if (!IsValid(HandActorClass))
 	{
 		HandActorClass = AHandActor::StaticClass();
@@ -69,7 +71,7 @@ void AHandManager::AddDeadHand(AHandActor* DeadHand)
 	DeadHands.Add(DeadHand);
 }
 
-TArray<AHandActor*> AHandManager::GetHandList()
+TArray<AHandActor*>& AHandManager::GetHandList()
 {
 	return HandList;
 }

Table/Source/Table/Private/MyGestureActions.cpp

@@ -0,0 +1,83 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+
+#include "MyGestureActions.h"
+#include "HandActor.h"
+#include "TableIconActor.h"
+
+
+void ARulerAction::UpdateAction_Implementation()
+{
+    if (!HandA) return;
+
+    // Update end point to current fingertip
+    Anchors.Add("End", HandA->Points[8]);
+
+    FVector Start = Anchors["Start"];
+    FVector End = Anchors["End"];
+
+    float RawDist = FVector::Dist(Start, End);
+    float Inches = GetSnappedInches(RawDist);
+
+    // Move line icon between start and end
+    if (Icons.IsValidIndex(0) && IsValid(Icons[0]))
+    {
+        Icons[0]->SetActorLocation((Start + End) * 0.5f);
+        // You'll add a SetEndpoints(Start, End) function to ATableIconLineActor later
+    }
+
+    // Move label icon to midpoint, slightly above table
+    if (Icons.IsValidIndex(1) && IsValid(Icons[1]))
+    {
+        FVector Mid = (Start + End) * 0.5f + FVector(0, 0, 5.f);
+        Icons[1]->SetActorLocation(Mid);
+        // Icons[1]->SetLabelText(FString::Printf(TEXT("%d\""), (int32)Inches));
+    }
+
+    
+}
+
+void ARulerAction::AbortAction_Implementation()
+{
+    Super::AbortAction();
+}
+
+void ARulerAction::AddExtension_Implementation(AGestureAction* OtherAction)
+{
+    return;
+}
+
+void ARulerAction::EndGesture_Implementation()
+{
+    Super::EndGesture();
+}
+
+void ARulerAction::InitAction_Implementation(AHandActor* Hand, const FGestureBinding& NewBinding, float NewUEUnitsPerUnit)
+{
+	Super::InitAction(Hand, NewBinding, NewUEUnitsPerUnit);
+	Anchors.Add("Start", Hand->Points[8]);
+
+    FActorSpawnParameters SpawnParams;
+    SpawnParams.Owner = GetOwner();
+    SpawnParams.SpawnCollisionHandlingOverride =
+        ESpawnActorCollisionHandlingMethod::AlwaysSpawn;
+   
+    AGestureAction* NewAction = nullptr;
+    /*AGestureAction* NewAction = GetWorld()->SpawnActor<AGestureAction>(
+        *ActionClassPtr,
+        FVector::ZeroVector,
+        FRotator::ZeroRotator,
+        SpawnParams);
+*/
+    if (!NewAction) return;
+
+
+}
+
+float ARulerAction::GetSnappedInches(float RawDistance) const
+{
+    if (UEUnitsPerUnit <= 0.f) return 0.f;
+    float Unit = RawDistance / UEUnitsPerUnit;
+    return FMath::RoundToFloat(Unit);  // snap to whole inches
+}
+

Table/Source/Table/Private/MyTableIconActors.cpp

@@ -0,0 +1,167 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+
+#include "MyTableIconActors.h"
+#include "KismetProceduralMeshLibrary.h"
+#include "TableIconActor.h"
+
+#include "Kismet/GameplayStatics.h"
+#include "Camera/CameraActor.h"
+
+ATableIconLineActor::ATableIconLineActor()
+{
+    MeshComponent = CreateDefaultSubobject<UProceduralMeshComponent>(TEXT("LineMesh"));
+    MeshComponent->SetupAttachment(RootComponent);
+    MeshComponent->SetCastShadow(false);
+}
+
+void ATableIconLineActor::SetEndpoints(const FVector& Start, const FVector& End)
+{
+    // Skip rebuild if nothing meaningful changed
+    if (Start.Equals(LastStart, 0.1f) && End.Equals(LastEnd, 0.1f)) return;
+    LastStart = Start;
+    LastEnd = End;
+
+    BuildLineMesh(Start, End);
+    if (bShowTickMarks) BuildTickMesh(Start, End);
+}
+
+void ATableIconLineActor::BuildLineMesh(const FVector& Start, const FVector& End)
+{
+    FVector Dir = (End - Start).GetSafeNormal();
+    FVector Up = FVector::UpVector;
+    FVector Right = FVector::CrossProduct(Dir, Up).GetSafeNormal() * LineThickness * 0.5f;
+
+    // A flat quad along the line, facing up
+    TArray<FVector>    Vertices;
+    TArray<int32>      Triangles;
+    TArray<FVector>    Normals;
+    TArray<FVector2D>  UVs;
+    TArray<FColor>     Colors;
+    TArray<FProcMeshTangent> Tangents;
+
+    Vertices.Add(Start - Right);  // 0
+    Vertices.Add(Start + Right);  // 1
+    Vertices.Add(End + Right);  // 2
+    Vertices.Add(End - Right);  // 3
+
+    Triangles = { 0, 1, 2,  0, 2, 3 };
+
+    for (int i = 0; i < 4; i++)
+    {
+        Normals.Add(FVector::UpVector);
+        UVs.Add(FVector2D(i < 2 ? 0.f : 1.f, i % 2 == 0 ? 0.f : 1.f));
+        Colors.Add(LineColor.ToFColor(true));
+        Tangents.Add(FProcMeshTangent(Dir, false));
+    }
+
+    // Section 0 = main line
+    MeshComponent->CreateMeshSection(0, Vertices, Triangles, Normals, UVs,
+        Colors, Tangents, false);
+}
+
+void ATableIconLineActor::BuildTickMesh(const FVector& Start, const FVector& End)
+{
+    if (UEUnitsPerUnit <= 0.f || TickInterval <= 0.f) return;
+
+    float TotalDist = FVector::Dist(Start, End);
+    float TickSpacing = TickInterval * UEUnitsPerUnit;
+    int32 NumTicks = FMath::FloorToInt(TotalDist / TickSpacing);
+
+    if (NumTicks <= 0)
+    {
+        MeshComponent->ClearMeshSection(1);
+        return;
+    }
+
+    FVector Dir = (End - Start).GetSafeNormal();
+    FVector Right = FVector::CrossProduct(Dir, FVector::UpVector).GetSafeNormal();
+
+    TArray<FVector>    Vertices;
+    TArray<int32>      Triangles;
+    TArray<FVector>    Normals;
+    TArray<FVector2D>  UVs;
+    TArray<FColor>     Colors;
+    TArray<FProcMeshTangent> Tangents;
+
+    float TickHalfLen = LineThickness * 3.f;
+    float TickHalfW = LineThickness * 0.5f;
+
+    for (int32 i = 1; i <= NumTicks; i++)
+    {
+        FVector Center = Start + Dir * (TickSpacing * i);
+        FVector A = Center - Right * TickHalfLen - Dir * TickHalfW;
+        FVector B = Center + Right * TickHalfLen - Dir * TickHalfW;
+        FVector C = Center + Right * TickHalfLen + Dir * TickHalfW;
+        FVector D = Center - Right * TickHalfLen + Dir * TickHalfW;
+
+        int32 Base = Vertices.Num();
+        Vertices.Add(A); Vertices.Add(B);
+        Vertices.Add(C); Vertices.Add(D);
+
+        Triangles.Add(Base);   Triangles.Add(Base + 1); Triangles.Add(Base + 2);
+        Triangles.Add(Base);   Triangles.Add(Base + 2); Triangles.Add(Base + 3);
+
+        for (int j = 0; j < 4; j++)
+        {
+            Normals.Add(FVector::UpVector);
+            UVs.Add(FVector2D::ZeroVector);
+            Colors.Add(LineColor.ToFColor(true));
+            Tangents.Add(FProcMeshTangent(Dir, false));
+        }
+    }
+
+    // Section 1 = tick marks
+    MeshComponent->CreateMeshSection(1, Vertices, Triangles, Normals, UVs,
+        Colors, Tangents, false);
+}
+
+
+
+
+
+ATableIconWidgetActor::ATableIconWidgetActor()
+{
+    PrimaryActorTick.bCanEverTick = true;
+
+    WidgetComponent = CreateDefaultSubobject<UWidgetComponent>(TEXT("WidgetComponent"));
+    WidgetComponent->SetupAttachment(RootComponent);
+    WidgetComponent->SetDrawAtDesiredSize(true);
+    WidgetComponent->SetWidgetSpace(EWidgetSpace::World);
+    WidgetComponent->SetCollisionEnabled(ECollisionEnabled::NoCollision);
+
+    WidgetInteraction = CreateDefaultSubobject<UWidgetInteractionComponent>(TEXT("WidgetInteraction"));
+    WidgetInteraction->SetupAttachment(RootComponent);
+    WidgetInteraction->InteractionSource = EWidgetInteractionSource::Custom;
+    WidgetInteraction->InteractionDistance = 10.f;
+}
+
+void ATableIconWidgetActor::Tick(float DeltaTime)
+{
+    Super::Tick(DeltaTime);
+
+    if (!bFaceCamera) return;
+
+    APlayerCameraManager* CamManager = UGameplayStatics::GetPlayerCameraManager(this, 0);
+    if (!CamManager) return;
+
+    FVector CamLocation = CamManager->GetCameraLocation();
+    FVector ToCamera = (CamLocation - GetActorLocation()).GetSafeNormal();
+
+    // Only rotate on yaw — keeps widget flat relative to table
+    ToCamera.Z = 0.f;
+    if (!ToCamera.IsNearlyZero())
+        SetActorRotation(ToCamera.ToOrientationRotator());
+}
+
+void ATableIconWidgetActor::SetWidgetClass(TSubclassOf<UUserWidget> InWidgetClass)
+{
+    if (!InWidgetClass) return;
+    WidgetComponent->SetWidgetClass(InWidgetClass);
+    WidgetComponent->InitWidget();
+}
+
+UUserWidget* ATableIconWidgetActor::GetWidget() const
+{
+    return WidgetComponent->GetUserWidgetObject();
+}

Table/Source/Table/Private/TableIconActor.cpp

@@ -0,0 +1,30 @@
+// Fill out your copyright notice in the Description page of Project Settings.
+
+
+#include "TableIconActor.h"
+
+// Sets default values
+ATableIconActor::ATableIconActor()
+{
+ 	// Set this actor to call Tick() every frame.  You can turn this off to improve performance if you don't need it.
+	PrimaryActorTick.bCanEverTick = false;
+
+}
+
+// Called when the game starts or when spawned
+void ATableIconActor::BeginPlay()
+{
+	Super::BeginPlay();
+	
+}
+
+void ATableIconActor::OnActionEnded_Implementation()
+{
+	if (!bPersistOnActionEnd) Destroy();
+}
+
+void ATableIconActor::OnActionUpdated_Implementation()
+{
+}
+
+

Table/Source/Table/Private/UserHUDWidget.cpp

@@ -107,8 +107,8 @@ FContentSource UUserHUDWidget::ParseContentSource(TSharedPtr<FJsonObject> Source
 
     if (ContentTypeString == TEXT("Image"))                 Source.ContentType = EContentSourceType::Image;
     else if (ContentTypeString == TEXT("PDF"))              Source.ContentType = EContentSourceType::PDF;
-    else if (ContentTypeString == TEXT("TextFile"))        Source.ContentType = EContentSourceType::TextFile;
-    else if (ContentTypeString == TEXT("InlineText"))      Source.ContentType = EContentSourceType::InlineText;
+    else if (ContentTypeString == TEXT("TextFile"))         Source.ContentType = EContentSourceType::TextFile;
+    else if (ContentTypeString == TEXT("InlineText"))       Source.ContentType = EContentSourceType::InlineText;
     else                                                    Source.ContentType = EContentSourceType::InlineText;
 
     return Source;