diff --git a/Table/Source/Table/Private/GestureDefinition.cpp b/Table/Source/Table/Private/GestureDefinition.cpp index eba038f..e4275f7 100644 --- a/Table/Source/Table/Private/GestureDefinition.cpp +++ b/Table/Source/Table/Private/GestureDefinition.cpp @@ -9,6 +9,104 @@ float UGestureDefinition::Evaluate(AHandActor* Hand) const return 0.f; } +int32 UGestureDefinition::GetFingerTipIndex(EHandFinger Finger) const +{ + switch (Finger) + { + case EHandFinger::Thumb: return 4; + case EHandFinger::Index: return 8; + case EHandFinger::Middle: return 12; + case EHandFinger::Ring: return 16; + case EHandFinger::Pinky: return 20; + } + return 0; +} + +int32 UGestureDefinition::GetFingerBaseIndex(EHandFinger Finger) const +{ + switch (Finger) + { + case EHandFinger::Thumb: return 1; // thumb base is special + case EHandFinger::Index: return 5; + case EHandFinger::Middle: return 9; + case EHandFinger::Ring: return 13; + case EHandFinger::Pinky: return 17; + } + return 0; +} + +const FVector& UGestureDefinition::GetPoint(const AHandActor* Hand, int32 Index) const +{ + return Hand->Points[Index]; +} + +float UGestureDefinition::GetHandScale(const AHandActor* Hand) const +{ + const TArray& P = Hand->Points; + return FVector::Dist(P[0], P[9]); // wrist -> middle base +} + +float UGestureDefinition::GetNormalizedDistance(const AHandActor* Hand, int32 A, int32 B) const +{ + float Dist = FVector::Dist(GetPoint(Hand, A), GetPoint(Hand, B)); + return Dist / GetHandScale(Hand); +} +float UGestureDefinition::GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const +{ + float Dist = GetNormalizedDistance( + Hand, + GetFingerTipIndex(A), + GetFingerTipIndex(B) + ); + + const float Threshold = 0.25f; + + return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f); +} + +float UGestureDefinition::GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const +{ + const TArray& P = Hand->Points; + + const FVector& Wrist = P[0]; + const FVector& Base = P[GetFingerBaseIndex(Finger)]; + const FVector& Tip = P[GetFingerTipIndex(Finger)]; + + float Dot = FVector::DotProduct( + (Tip - Base).GetSafeNormal(), + (Base - Wrist).GetSafeNormal() + ); + + return FMath::Clamp(Dot, 0.f, 1.f); +} + +float UGestureDefinition::GetFingerCurledScore(const AHandActor* Hand, EHandFinger Finger) const +{ + return 1.0f - GetFingerExtendedScore(Hand, Finger); +} + +float UGestureDefinition::GetFingersAverage(const AHandActor* Hand, const TArray& Fingers, bool bExtended) const +{ + float Sum = 0.f; + + for (EHandFinger Finger : Fingers) + { + Sum += bExtended + ? GetFingerExtendedScore(Hand, Finger) + : GetFingerCurledScore(Hand, Finger); + } + + return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f; +} + +FVector UGestureDefinition::GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const +{ + const FVector& Base = GetPoint(Hand, GetFingerBaseIndex(Finger)); + const FVector& Tip = GetPoint(Hand, GetFingerTipIndex(Finger)); + + return (Tip - Base).GetSafeNormal(); +} + float UGesture_PinchPoint::Evaluate(AHandActor* Hand) const { float Pinch = GetPinchScore(Hand, EHandFinger::Thumb, EHandFinger::Index); diff --git a/Table/Source/Table/Private/GestureManagerComponent.cpp b/Table/Source/Table/Private/GestureManagerComponent.cpp index 72be9b0..8fe1a36 100644 --- a/Table/Source/Table/Private/GestureManagerComponent.cpp +++ b/Table/Source/Table/Private/GestureManagerComponent.cpp @@ -25,6 +25,8 @@ void UGestureManagerComponent::BeginPlay() HandManager->OnHandAdded.AddDynamic(this, &UGestureManagerComponent::OnHandAdded); HandManager->OnHandRemoved.AddDynamic(this, &UGestureManagerComponent::OnHandRemoved); + + LoadBindings("C:/git/Table/Test/Gestures/gesture_config.json"); } void UGestureManagerComponent::EvaluateHands() @@ -64,6 +66,107 @@ void UGestureManagerComponent::OnHandRemoved(AHandActor* Hand) Hand->ResetGesture(); } +void UGestureManagerComponent::LoadBindings(const FString& FilePath) +{ + FString JsonString; + + if (!FFileHelper::LoadFileToString(JsonString, *FilePath)) + { + return; + } + + TSharedPtr Root; + TSharedRef> Reader = TJsonReaderFactory<>::Create(JsonString); + + if (!FJsonSerializer::Deserialize(Reader, Root) || !Root.IsValid()) + return; + + const TArray>* BindingsArray; + + if (Root->TryGetArrayField(TEXT("bindings"), BindingsArray)) + { + for (const TSharedPtr& Value : *BindingsArray) + { + TSharedPtr Obj = Value->AsObject(); + if (!Obj.IsValid()) continue; + + FGestureBinding Binding; + + Obj->TryGetStringField(TEXT("action"), Binding.ActionName); + Obj->TryGetStringField(TEXT("left"), Binding.LeftGesture); + Obj->TryGetStringField(TEXT("right"), Binding.RightGesture); + + const TArray>* ExtArray; + if (Obj->TryGetArrayField(TEXT("extensions"), ExtArray)) + { + for (const TSharedPtr& ExtVal : *ExtArray) + { + TSharedPtr ExtObj = ExtVal->AsObject(); + if (!ExtObj.IsValid()) continue; + + FExtendedGestureBinding Ext; + + ExtObj->TryGetStringField(TEXT("gesture"), Ext.GestureName); + ExtObj->TryGetStringField(TEXT("action"), Ext.GestureAction); + + Binding.ExentdedGestures.Add(Ext); + } + } + + GestureBindings.Add(Binding); + } + } +} + +void UGestureManagerComponent::Initialize(UDataTable* InGestureTable, UDataTable* InActionTable) +{ + GestureDefinitionTable = InGestureTable; + GestureActionTable = InActionTable; + + BuildDefinitionMap(); + BuildActionMap(); +} + +void UGestureManagerComponent::BuildDefinitionMap() +{ + if (!GestureDefinitionTable) return; + + static const FString Context = TEXT("GestureDefinitionContext"); + + TArray Rows; + GestureDefinitionTable->GetAllRows(Context, Rows); + + for (FGestureDefinitionRow* Row : Rows) + { + if (!Row || !Row->DefinitionClass) continue; + + UGestureDefinition* Def = NewObject(this, Row->DefinitionClass); + + DefinitionMap.Add(Def->GestureName, Def); + } +} + +void UGestureManagerComponent::BuildActionMap() +{ + if (!GestureActionTable) return; + + static const FString Context = TEXT("GestureActionContext"); + + TArray Rows; + GestureActionTable->GetAllRows(Context, Rows); + + for (FGestureActionRow* Row : Rows) + { + if (!Row || !Row->ActionClass) continue; + + ActionClassMap.Add( + Row->ActionClass->GetName(), + Row->ActionClass + + ); + } +} + void UGestureManagerComponent::CheckGestureDefinitions(TArray& Hands) { const float MinScoreThreshold = 0.5f; diff --git a/Table/Source/Table/Private/HandManager.cpp b/Table/Source/Table/Private/HandManager.cpp index 7f91e12..0fb206d 100644 --- a/Table/Source/Table/Private/HandManager.cpp +++ b/Table/Source/Table/Private/HandManager.cpp @@ -10,6 +10,7 @@ AHandManager::AHandManager() // Set this actor to call Tick() every frame. You can turn this off to improve performance if you don't need it. PrimaryActorTick.bCanEverTick = true; GestureManager = CreateDefaultSubobject(TEXT("GestureManager")); + GestureManager->Initialize(GestureDefinitionTable, GestureActionTable); } // Called when the game starts or when spawned diff --git a/Table/Source/Table/Public/GestureDefinition.h b/Table/Source/Table/Public/GestureDefinition.h index a4c3635..08f1d69 100644 --- a/Table/Source/Table/Public/GestureDefinition.h +++ b/Table/Source/Table/Public/GestureDefinition.h @@ -33,107 +33,23 @@ public: UPROPERTY(BlueprintReadWrite) FString GestureName; - UFUNCTION() + + // This checks the gesture and returns an aproximate scor of how well the Gesture was exectuded, needs to be overwritten in child classes. + UFUNCTION(BlueprintCallable) virtual float Evaluate(AHandActor* Hand) const; - int32 GetFingerTipIndex(EHandFinger Finger) const - { - switch (Finger) - { - case EHandFinger::Thumb: return 4; - case EHandFinger::Index: return 8; - case EHandFinger::Middle: return 12; - case EHandFinger::Ring: return 16; - case EHandFinger::Pinky: return 20; - } - return 0; - } - - int32 GetFingerBaseIndex(EHandFinger Finger) const - { - switch (Finger) - { - case EHandFinger::Thumb: return 1; // thumb base is special - case EHandFinger::Index: return 5; - case EHandFinger::Middle: return 9; - case EHandFinger::Ring: return 13; - case EHandFinger::Pinky: return 17; - } - return 0; - } - - const FVector& GetPoint(const AHandActor* Hand, int32 Index) const - { - return Hand->Points[Index]; - } - - float GetHandScale(const AHandActor* Hand) const - { - const TArray& P = Hand->Points; - return FVector::Dist(P[0], P[9]); // wrist -> middle base - } - - float GetNormalizedDistance(const AHandActor* Hand, int32 A, int32 B) const - { - float Dist = FVector::Dist(GetPoint(Hand, A), GetPoint(Hand, B)); - return Dist / GetHandScale(Hand); - } - - float GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const - { - float Dist = GetNormalizedDistance( - Hand, - GetFingerTipIndex(A), - GetFingerTipIndex(B) - ); - - const float Threshold = 0.25f; - - return 1.0f - FMath::Clamp(Dist / Threshold, 0.f, 1.f); - } - - float GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const - { - const TArray& P = Hand->Points; - - const FVector& Wrist = P[0]; - const FVector& Base = P[GetFingerBaseIndex(Finger)]; - const FVector& Tip = P[GetFingerTipIndex(Finger)]; - - float Dot = FVector::DotProduct( - (Tip - Base).GetSafeNormal(), - (Base - Wrist).GetSafeNormal() - ); - - return FMath::Clamp(Dot, 0.f, 1.f); - } - - float GetFingerCurledScore(const AHandActor* Hand, EHandFinger Finger) const - { - return 1.0f - GetFingerExtendedScore(Hand, Finger); - } - - float GetFingersAverage(const AHandActor* Hand, const TArray& Fingers, bool bExtended) const - { - float Sum = 0.f; - - for (EHandFinger Finger : Fingers) - { - Sum += bExtended - ? GetFingerExtendedScore(Hand, Finger) - : GetFingerCurledScore(Hand, Finger); - } - - return Fingers.Num() > 0 ? Sum / Fingers.Num() : 0.f; - } - - FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const - { - const FVector& Base = GetPoint(Hand, GetFingerBaseIndex(Finger)); - const FVector& Tip = GetPoint(Hand, GetFingerTipIndex(Finger)); - - return (Tip - Base).GetSafeNormal(); - } +protected: + // Helper functions to more easily check Gestures + int32 GetFingerTipIndex(EHandFinger Finger) const; + int32 GetFingerBaseIndex(EHandFinger Finger) const; + float GetHandScale(const AHandActor* Hand) const; + float GetNormalizedDistance(const AHandActor* Hand, int32 A, int32 B) const; + float GetPinchScore(const AHandActor* Hand, EHandFinger A, EHandFinger B) const; + float GetFingerExtendedScore(const AHandActor* Hand, EHandFinger Finger) const; + float GetFingerCurledScore(const AHandActor* Hand, EHandFinger Finger) const; + float GetFingersAverage(const AHandActor* Hand, const TArray& Fingers, bool bExtended) const; + FVector GetFingerDirection(const AHandActor* Hand, EHandFinger Finger) const; + const FVector& GetPoint(const AHandActor* Hand, int32 Index) const; }; UCLASS() diff --git a/Table/Source/Table/Public/GestureManagerComponent.h b/Table/Source/Table/Public/GestureManagerComponent.h index 44ccc01..22bffc4 100644 --- a/Table/Source/Table/Public/GestureManagerComponent.h +++ b/Table/Source/Table/Public/GestureManagerComponent.h @@ -49,6 +49,11 @@ public: // Populated on BeginPlay from JSON UPROPERTY(BlueprintReadOnly, Category = "Gesture") TArray GestureBindings; + void LoadBindings(const FString& FilePath); + + void Initialize(UDataTable* InGestureTable, UDataTable* InActionTable); + void BuildDefinitionMap(); + void BuildActionMap(); // Populated on BeginPlay from the tables UPROPERTY(BlueprintReadOnly, Category = "Gesture") @@ -57,8 +62,6 @@ public: UPROPERTY(BlueprintReadOnly, Category = "Gesture") TMap> ActionClassMap; - - private: // GestureManager.h — add these UPROPERTY(EditDefaultsOnly, Category = "Gesture") diff --git a/Table/Source/Table/Public/HandManager.h b/Table/Source/Table/Public/HandManager.h index eecea9e..c77e83d 100644 --- a/Table/Source/Table/Public/HandManager.h +++ b/Table/Source/Table/Public/HandManager.h @@ -71,6 +71,13 @@ public: const TArray& GetHandList() const { return HandList; } + // GestureManager.h — add these + UPROPERTY(EditDefaultsOnly, Category = "Gesture") + UDataTable* GestureDefinitionTable; + + UPROPERTY(EditDefaultsOnly, Category = "Gesture") + UDataTable* GestureActionTable; + private: void RemoveDeadHands(); void UpdateHands();