Fix camera capture reliability and rewrite tracking for available mediapipe/hardware

source/Tbd/TrackingController.py

@@ -1,7 +1,10 @@
 import numpy as np
 import cv2
+import os
 from PySide6.QtCore import QObject, QTimer
 import mediapipe as mp
+from mediapipe.tasks.python import BaseOptions
+from mediapipe.tasks.python.vision import HandLandmarker, HandLandmarkerOptions, RunningMode
 from Tbd.helper import Hand2D
 from collections import defaultdict
 from Helpers.HandUdpSender import HandUdpSender
@@ -9,6 +12,8 @@ from Helpers.HandUdpSender import HandUdpSender
 import struct
 import time
 
+MODEL_PATH = os.path.join(os.path.dirname(__file__), "models", "hand_landmarker.task")
+
 class TrackingController(QObject):
     def __init__(self, cameraList, P_mats, logger=None, tick_ms=30):
         super().__init__()
@@ -25,16 +30,20 @@ class TrackingController(QObject):
 
         self.running = False
 
-        # init mediapipe here (or inject it)
-
-        self.mp_hands = mp.solutions.hands
-        self.hands = self.mp_hands.Hands(
-            static_image_mode=False,
-            max_num_hands=4,
-            model_complexity=1,
-            min_detection_confidence=0.5,
+        # MediaPipe's legacy solutions.hands API no longer exists on the
+        # installed mediapipe version, so hand detection runs on the newer
+        # Tasks API instead (IMAGE mode: each camera frame is detected
+        # independently, since frames from different cameras aren't a single
+        # monotonic video stream that VIDEO mode requires).
+        options = HandLandmarkerOptions(
+            base_options=BaseOptions(model_asset_path=MODEL_PATH),
+            running_mode=RunningMode.IMAGE,
+            num_hands=4,
+            min_hand_detection_confidence=0.5,
+            min_hand_presence_confidence=0.5,
             min_tracking_confidence=0.5,
         )
+        self.detector = HandLandmarker.create_from_options(options)
 
     def start(self):
         # Guard: need projections
@@ -45,6 +54,11 @@ class TrackingController(QObject):
 
         if self.running:
             return
+
+        for camera in self.cameraList:
+            if not camera.open():
+                if self.logger: self.logger.log(f"[Tracking] cam{camera.index}: failed to open, will be skipped.")
+
         self.running = True
         self.timer.start(self.tick_ms)
         if self.logger: self.logger.log("[Tracking] Started")
@@ -54,6 +68,10 @@ class TrackingController(QObject):
             return
         self.running = False
         self.timer.stop()
+
+        for camera in self.cameraList:
+            camera.close()
+
         if self.logger: self.logger.log("[Tracking] Stopped")
 
     def _tick(self):
@@ -81,30 +99,33 @@ class TrackingController(QObject):
     def _detectHands_all_cameras(self):
         out = []
         for camera in self.cameraList:
+            if camera.capture is None:
+                continue
+
             ok, frameBGR = camera.capture.read()
             if not ok or frameBGR is None:
                 continue
 
             h, w = frameBGR.shape[:2]
             frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
-            frameRGB.flags.writeable = False
 
-            hand_tracking_result = self.hands.process(frameRGB)
-            if not hand_tracking_result.multi_hand_landmarks:
+            mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frameRGB)
+            result = self.detector.detect(mp_image)
+
+            if not result.hand_landmarks:
                 continue
 
-            handed = hand_tracking_result.multi_handedness
-            for hi, lm_list in enumerate(hand_tracking_result.multi_hand_landmarks):
+            for hi, lm_list in enumerate(result.hand_landmarks):
                 landmarks_px = np.array(
-                    [[lm.x * w, lm.y * h] for lm in lm_list.landmark],
+                    [[lm.x * w, lm.y * h] for lm in lm_list],
                     dtype=np.float32
                 )
 
                 label, score = "Unknown", 0.0
-                if hi < len(handed) and handed[hi].classification:
-                    label = handed[hi].classification[0].label
-                    score = handed[hi].classification[0].score
-                
+                if hi < len(result.handedness) and result.handedness[hi]:
+                    label = result.handedness[hi][0].category_name
+                    score = result.handedness[hi][0].score
+
                 out.append(Hand2D(
                     camera_id=camera.index,
                     handedness=label,

source/Tbd/helper.py

@@ -2,6 +2,7 @@ import numpy as np
 import json
 import cv2
 import threading
+import time
 
 from dataclasses import dataclass, field, asdict
 from typing import List, Optional, Tuple, Dict
@@ -122,7 +123,39 @@ class CameraObject():
         #self.capture = cv2.VideoCapture(self.index, cv2.CAP_DSHOW)
         self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2048.0)
         self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1536.0)
- 
+
+    def open(self, width: int = 1920, height: int = 1080, max_attempts: int = 5) -> bool:
+        """
+        Open this camera on demand via MSMF. Only MSMF has been able to open all
+        6 cameras on this rig at all (DSHOW hits a hard concurrent-instance limit
+        around 2-3 devices on this hardware) - the tradeoff is MSMF cameras should
+        not be kept open simultaneously in large numbers, so callers are expected
+        to open() right before use and close() right after.
+        """
+        if self.capture is not None:
+            return True
+
+        for attempt in range(1, max_attempts + 1):
+            capture = cv2.VideoCapture(self.index, cv2.CAP_MSMF)
+            if capture.isOpened():
+                capture.set(cv2.CAP_PROP_FRAME_WIDTH, width)
+                capture.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
+                capture.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
+                self.capture = capture
+                self.pxWidth = width
+                self.pxHeight = height
+                return True
+
+            capture.release()
+            time.sleep(0.3 * attempt)
+
+        return False
+
+    def close(self):
+        if self.capture is not None:
+            self.capture.release()
+            self.capture = None
+
 
 @dataclass
 class Hand2D():