Added continous hand tracking

source/Tbd/TrackingController.py

@@ -32,18 +32,25 @@ class TrackingController(QObject):
 
         # MediaPipe's legacy solutions.hands API no longer exists on the
         # installed mediapipe version, so hand detection runs on the newer
-        # Tasks API instead (IMAGE mode: each camera frame is detected
-        # independently, since frames from different cameras aren't a single
-        # monotonic video stream that VIDEO mode requires).
-        options = HandLandmarkerOptions(
-            base_options=BaseOptions(model_asset_path=MODEL_PATH),
-            running_mode=RunningMode.IMAGE,
-            num_hands=4,
-            min_hand_detection_confidence=0.5,
-            min_hand_presence_confidence=0.5,
-            min_tracking_confidence=0.5,
-        )
-        self.detector = HandLandmarker.create_from_options(options)
+        # Tasks API instead. Each camera gets its own HandLandmarker running
+        # in VIDEO mode, fed only that camera's own frames with a monotonically
+        # increasing per-camera timestamp - this lets MediaPipe track hands
+        # frame-to-frame per view (much less jitter than re-detecting blind
+        # every frame), which wouldn't be valid if frames from different
+        # camera viewpoints were interleaved through a single VIDEO-mode
+        # detector.
+        self._frame_counter = defaultdict(int)
+        self.detectors = {}
+        for camera in self.cameraList:
+            options = HandLandmarkerOptions(
+                base_options=BaseOptions(model_asset_path=MODEL_PATH),
+                running_mode=RunningMode.VIDEO,
+                num_hands=12,
+                min_hand_detection_confidence=0.5,
+                min_hand_presence_confidence=0.5,
+                min_tracking_confidence=0.5,
+            )
+            self.detectors[camera.index] = HandLandmarker.create_from_options(options)
 
     def start(self):
         # Guard: need projections
@@ -72,6 +79,9 @@ class TrackingController(QObject):
         for camera in self.cameraList:
             camera.close()
 
+        for detector in self.detectors.values():
+            detector.close()
+
         if self.logger: self.logger.log("[Tracking] Stopped")
 
     def _tick(self):
@@ -102,6 +112,10 @@ class TrackingController(QObject):
             if camera.capture is None:
                 continue
 
+            detector = self.detectors.get(camera.index)
+            if detector is None:
+                continue
+
             ok, frameBGR = camera.capture.read()
             if not ok or frameBGR is None:
                 continue
@@ -110,7 +124,10 @@ class TrackingController(QObject):
             frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
 
             mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frameRGB)
-            result = self.detector.detect(mp_image)
+
+            self._frame_counter[camera.index] += 1
+            timestamp_ms = self._frame_counter[camera.index] * self.tick_ms
+            result = detector.detect_for_video(mp_image, timestamp_ms)
 
             if not result.hand_landmarks:
                 continue