0
\$\begingroup\$

I'm new to C# programming and have been working on a Unity project that incorporates gesture recognition using the PDollarGestureRecognizer library and Unity XR Hands package. My application is designed to capture hand movements through an XR input source, recognize these gestures, and operates in two distinct modes: a "creative mode" for capturing new gestures, and a "recognition mode" for identifying gestures from a set.

While I'm satisfied with the basic functionality, I'm aware that there's always room for improvement, particularly regarding efficiency and code clarity. I'm keen to receive any advice, critique, or suggestions on refining my code.

Code:

using PDollarGestureRecognizer;
using UnityEngine;
using UnityEngine.XR.Hands;
using System.Collections.Generic;
using System.IO;

public class ManyGesture : MonoBehaviour
{
[SerializeField] private XRHandSubsystem _handSubsystem;
[SerializeField] private float _handScalingFactor = 1.0f;
private List<Gesture> _trainingSet = new List<Gesture>();
[SerializeField] private float _recognitionThreshold = 0.9f;
private List<List<Vector3>> _tempGestureFrames = new List<List<Vector3>>();

private bool _isCapturingDynamicGesture = false;
private int _gestureCount = 0;
private bool _isInCreativeMode = false;
[SerializeField] private int _framesThreshold = 250;

void Start()
{
    LoadGestures();
    InitializeHandSubsystem();
}

void Update()
{
    HandleGestureCapture();
}

private void LoadGestures()
{
    string[] gestureFiles = Directory.GetFiles(Application.persistentDataPath, "*.xml");
    foreach (var item in gestureFiles)
    {
        _trainingSet.Add(GestureIO.ReadGestureFromFile(item));
    }
    Debug.Log("Gestures loaded.");
}

private void InitializeHandSubsystem()
{
    var handSubsystems = new List<XRHandSubsystem>();
    SubsystemManager.GetSubsystems(handSubsystems);
    if (handSubsystems.Count > 0)
    {
        _handSubsystem = handSubsystems[0];
        Debug.Log("HandSubsystem found.");
    }
}

private void HandleGestureCapture()
{
    if (_isInCreativeMode)
    {
        CaptureModeUpdate();
    }
    else
    {
        RecognitionModeUpdate();
    }
}

private void CaptureModeUpdate()
{
    if (Input.GetKeyDown(KeyCode.S) && !_isCapturingDynamicGesture)
    {
        StartGestureCapture();
    }
    else if (Input.GetKeyUp(KeyCode.E) && _isCapturingDynamicGesture)
    {
        EndGestureCapture();
    }
    else if (_isCapturingDynamicGesture)
    {
        UpdateGestureCapture();
    }
}

private void RecognitionModeUpdate()
{
    List<Vector3> currentFrame = GetCurrentJointPositions(_handSubsystem.rightHand);
    if (currentFrame == null) return;

    AddCurrentFrame(currentFrame);

    if (ShouldAttemptRecognition())
    {
        RecognizeAndResetGesture();
    }
}

private void AddCurrentFrame(List<Vector3> currentFrame)
{
    if (!_isCapturingDynamicGesture)
    {
        _isCapturingDynamicGesture = true;
        _tempGestureFrames = new List<List<Vector3>>();
    }
    _tempGestureFrames.Add(currentFrame);
}

private void RecognizeAndResetGesture()
{
    var combinedGesturePoints = CombineGesturePoints();
    var currentGesture = ConvertToGesture(combinedGesturePoints);
    RecognizeGesture(currentGesture);
    _isCapturingDynamicGesture = false;
    _tempGestureFrames.Clear();
}

bool ShouldAttemptRecognition()
{
    return _tempGestureFrames.Count >= _framesThreshold;
}

private void StartGestureCapture()
{
    _isCapturingDynamicGesture = true;
    _tempGestureFrames.Clear();
    Debug.Log("Starting gesture capture.");
}

private void EndGestureCapture()
{
    _isCapturingDynamicGesture = false;
    var combinedGesturePoints = CombineGesturePoints();
    var capturedGesture = ConvertToGesture(combinedGesturePoints);
    capturedGesture.Name = $"Gesture_{++_gestureCount}";
    SaveGesture(capturedGesture);
}

private List<Vector3> CombineGesturePoints()
{
    List<Vector3> combinedGesturePoints = new List<Vector3>();
    foreach (var frame in _tempGestureFrames)
    {
        combinedGesturePoints.AddRange(frame);
    }
    return combinedGesturePoints;
}

private void UpdateGestureCapture()
{
    var currentFrame = GetCurrentJointPositions(_handSubsystem.rightHand);
    if (currentFrame != null)
    {
        _tempGestureFrames.Add(currentFrame);
    }
}

private Gesture ConvertToGesture(List<Vector3> positions)
{
    Point[] points = new Point[positions.Count];
    for (int i = 0; i < positions.Count; i++)
    {
        points[i] = new Point(positions[i].x, positions[i].y, 0); // Assuming Z-coordinate is 0 for 2D recognition
    }
    return new Gesture(points);
}

private void RecognizeGesture(Gesture gesture)
{
    Result result = PointCloudRecognizer.Classify(gesture, _trainingSet.ToArray());
    if (result.Score > _recognitionThreshold)
    {
        Debug.Log($"Gesture recognized: {result.GestureClass} with accuracy {result.Score}");
    }
}

private void SaveGesture(Gesture gesture)
{
    string fileName = Path.Combine(Application.persistentDataPath, $"{gesture.Name}.xml");
    GestureIO.WriteGesture(gesture.Points, gesture.Name, fileName);
    Debug.Log($"Gesture {gesture.Name} saved.");
}

private List<Vector3> GetCurrentJointPositions(XRHand hand)
{
    if (!hand.isTracked) return null;

    List<Vector3> jointPositions = new List<Vector3>();
    XRHandJointID[] specificJoints = {
        XRHandJointID.IndexTip,
        XRHandJointID.MiddleTip,
        XRHandJointID.RingTip,
        XRHandJointID.LittleTip,
        XRHandJointID.Palm,
        XRHandJointID.Wrist
    };

    foreach (XRHandJointID jointId in specificJoints)
    {
        XRHandJoint jointData = hand.GetJoint(jointId);
        if (jointData.TryGetPose(out Pose pose))
        {
            Vector3 localPosition = transform.InverseTransformPoint(pose.position);
            localPosition /= _handScalingFactor;
            jointPositions.Add(localPosition);
        }
    }
    return jointPositions;
}

public void ToggleCreativeMode()
{
    _isInCreativeMode = !_isInCreativeMode;
    Debug.Log($"Creative Mode: {(_isInCreativeMode ? "Active" : "Inactive")}");
    }
}

Areas for Feedback:

  1. Efficency: Concerned about continuously capturing hand joint data in Update() and evaluating gestures after accumulating a set number of frames.
  2. Code Clarity & Best Practices: Looking for advice on making the code more concise and aligning it with C# and Unity best practices, particularly for XR input handling.

Any insights or suggestions would be appreciated!

\$\endgroup\$

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Browse other questions tagged or ask your own question.