Commit for Gitea

source/Helpers/CHArUco_version_test.py

@@ -0,0 +1,5 @@
+import cv2
+print("OpenCV:", cv2.__version__)
+print("has aruco:", hasattr(cv2, "aruco"))
+print("has CharucoBoard:", hasattr(cv2.aruco, "CharucoBoard"))
+print("has ArucoDetector:", hasattr(cv2.aruco, "ArucoDetector"))

source/Helpers/ChArUco.py

@@ -0,0 +1,32 @@
+import cv2
+import numpy as np
+
+# ---- Board parameters (CHANGE THESE ONLY IF YOU REPRINT) ----
+squares_x = 6        # number of chessboard squares in X
+squares_y = 9        # number of chessboard squares in Y
+square_length = 25  # mm
+marker_length = 18  # mm (must be < square_length)
+dictionary_id = cv2.aruco.DICT_4X4_50
+dpi = 300            # print DPI
+# ------------------------------------------------------------
+
+dictionary = cv2.aruco.getPredefinedDictionary(dictionary_id)
+board = cv2.aruco.CharucoBoard(
+    (squares_x, squares_y),
+    square_length,
+    marker_length,
+    dictionary
+)
+
+# Convert physical size (mm) to pixels for printing
+mm_to_inch = 1 / 25.4
+width_mm = squares_x * square_length
+height_mm = squares_y * square_length
+
+width_px = int(width_mm * mm_to_inch * dpi)
+height_px = int(height_mm * mm_to_inch * dpi)
+
+img = board.generateImage((width_px, height_px))
+img = 255 - img
+cv2.imwrite("H:\Table\charuco_A4.png", img)
+print("Saved charuco_A4.png")

source/Helpers/ChArUco_confidence_Test.py

@@ -0,0 +1,220 @@
+"""
+ChArUco detection confidence test (OpenCV)
+
+What it does:
+- Opens one or multiple cameras
+- Detects ArUco markers and interpolated ChArUco corners
+- Draws overlays
+- Prints a simple "confidence" score per camera:
+    markers_found, charuco_corners_found, and a normalized confidence value
+
+Requirements:
+- OpenCV built with aruco module (opencv-contrib-python)
+    pip install opencv-contrib-python
+"""
+
+import cv2
+import numpy as np
+import time
+
+# ----------------------------
+# USER SETTINGS (match your printed board!)
+# ----------------------------
+CAM_IDS = [0, 2]                 # set to [0] for single camera test, or [0,2,3,...]
+USE_DSHOW_ON_WINDOWS = True      # good for Windows
+RESOLUTION = (3264, 2448)        # (width, height) if your cameras support it; else set None
+DICT_ID = cv2.aruco.DICT_4X4_50
+
+SQUARES_X = 6                    # number of chessboard squares in X
+SQUARES_Y = 9                    # number of chessboard squares in Y
+SQUARE_LEN_MM = 25.0             # square size in mm
+MARKER_LEN_MM = 18.0             # marker size in mm (must be < square)
+
+# Confidence thresholds (tune if needed)
+MIN_MARKERS_OK = 4               # markers to consider "good"
+MIN_CHARUCO_OK = 15              # charuco corners to consider "good"
+
+# Display
+WINDOW_SCALE = 0.5               # downscale for display if res is huge (0.5 = half size)
+PRINT_EVERY_SEC = 0.5
+# ----------------------------
+
+
+def open_camera(cam_id: int):
+    backend = cv2.CAP_DSHOW if (USE_DSHOW_ON_WINDOWS and hasattr(cv2, "CAP_DSHOW")) else 0
+    cap = cv2.VideoCapture(cam_id, backend)
+    if not cap.isOpened():
+        return None
+
+    if RESOLUTION is not None:
+        w, h = RESOLUTION
+        cap.set(cv2.CAP_PROP_FRAME_WIDTH, float(w))
+        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, float(h))
+
+    return cap
+
+
+def compute_confidence(markers: int, charuco: int) -> float:
+    """
+    Simple normalized confidence heuristic:
+    - markers contribute up to MIN_MARKERS_OK
+    - charuco corners contribute up to MIN_CHARUCO_OK
+    """
+    m = min(markers / max(MIN_MARKERS_OK, 1), 1.0)
+    c = min(charuco / max(MIN_CHARUCO_OK, 1), 1.0)
+    # Weighted: charuco corners matter more for calibration quality
+    return 0.35 * m + 0.65 * c
+
+
+def main():
+    dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID)
+    board = cv2.aruco.CharucoBoard(
+        (SQUARES_X, SQUARES_Y),
+        SQUARE_LEN_MM,
+        MARKER_LEN_MM,
+        dictionary
+    )
+
+    # Detector parameters
+    detector_params = cv2.aruco.DetectorParameters()
+    # You can tweak these if detection is unstable:
+    # detector_params.adaptiveThreshWinSizeMin = 3
+    # detector_params.adaptiveThreshWinSizeMax = 23
+    # detector_params.adaptiveThreshWinSizeStep = 10
+
+    detector = cv2.aruco.ArucoDetector(dictionary, detector_params)
+
+    caps = {}
+    for cam_id in CAM_IDS:
+        cap = open_camera(cam_id)
+        if cap is None:
+            print(f"[ERR] Could not open camera {cam_id}")
+        else:
+            caps[cam_id] = cap
+            print(f"[OK] Opened camera {cam_id}")
+
+    if not caps:
+        print("No cameras opened. Exiting.")
+        return
+
+    last_print = 0.0
+
+    print("\nControls:")
+    print("  ESC = quit")
+    print("  Space = print one-shot stats immediately\n")
+
+    try:
+        while True:
+            frames_vis = []
+            stats = {}
+
+            for cam_id, cap in caps.items():
+                ok, frame = cap.read()
+                if not ok or frame is None:
+                    stats[cam_id] = (0, 0, 0.0)
+                    continue
+
+                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+                gray = 255 - gray
+                # Detect markers
+                corners, ids, rejected = detector.detectMarkers(gray)
+                markers_found = 0 if ids is None else len(ids)
+
+                # Draw markers
+                vis = frame.copy()
+                if ids is not None:
+                    cv2.aruco.drawDetectedMarkers(vis, corners, ids)
+
+                # Interpolate ChArUco corners (requires some markers found)
+                charuco_found = 0
+                if ids is not None and len(ids) > 0:
+                    charuco_detector = cv2.aruco.CharucoDetector(board)
+
+                    charuco_corners = None
+                    charuco_ids = None
+
+                    if ids is not None and len(ids) > 0:
+                        res = charuco_detector.detectBoard(gray)
+
+                        # OpenCV versions differ in what they return; handle both safely.
+                        # Common patterns:
+                        #   (charucoCorners, charucoIds, markerCorners, markerIds)
+                        #   (charucoCorners, charucoIds, rejectedMarkerCandidates)
+                        #   (charucoCorners, charucoIds, ...)
+                        charuco_corners = res[0] if len(res) > 0 else None
+                        charuco_ids = res[1] if len(res) > 1 else None
+                    charuco_found = 0
+                    if charuco_ids is not None:
+                        charuco_found = len(charuco_ids)
+                        cv2.aruco.drawDetectedCornersCharuco(vis, charuco_corners, charuco_ids)
+
+                conf = compute_confidence(markers_found, charuco_found)
+                stats[cam_id] = (markers_found, charuco_found, conf)
+
+                # Overlay text
+                h, w = vis.shape[:2]
+                lines = [
+                    f"Cam {cam_id}",
+                    f"Markers: {markers_found}",
+                    f"ChArUco corners: {charuco_found}",
+                    f"Confidence: {conf:.2f}",
+                    "GOOD" if (markers_found >= MIN_MARKERS_OK and charuco_found >= MIN_CHARUCO_OK) else "MOVE / LIGHT / FOCUS"
+                ]
+
+                y = 30
+                for line in lines:
+                    cv2.putText(vis, line, (20, y), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0) if "GOOD" in line else (0, 200, 255), 2)
+                    y += 32
+
+                # Downscale for display
+                if WINDOW_SCALE != 1.0:
+                    vis = cv2.resize(vis, None, fx=WINDOW_SCALE, fy=WINDOW_SCALE, interpolation=cv2.INTER_AREA)
+
+                frames_vis.append(vis)
+
+            # Combine displays
+            if frames_vis:
+                # Stack horizontally; if many cams, wrap to multiple rows
+                max_per_row = 3
+                rows = []
+                for i in range(0, len(frames_vis), max_per_row):
+                    row = frames_vis[i:i + max_per_row]
+                    # pad heights
+                    max_h = max(img.shape[0] for img in row)
+                    padded = []
+                    for img in row:
+                        if img.shape[0] < max_h:
+                            pad = max_h - img.shape[0]
+                            img = cv2.copyMakeBorder(img, 0, pad, 0, 0, cv2.BORDER_CONSTANT, value=(0, 0, 0))
+                        padded.append(img)
+                    rows.append(np.hstack(padded))
+                grid = np.vstack(rows)
+
+                cv2.imshow("ChArUco Confidence Test", grid)
+
+            key = cv2.waitKey(1) & 0xFF
+            now = time.time()
+
+            if key == 27:  # ESC
+                break
+
+            if key == 32:  # Space
+                last_print = 0  # force print now
+
+            if now - last_print >= PRINT_EVERY_SEC:
+                last_print = now
+                # Print compact stats
+                msg = " | ".join(
+                    f"cam{cid}: M={m} C={c} conf={conf:.2f}"
+                    for cid, (m, c, conf) in sorted(stats.items())
+                )
+                print(msg)
+
+    finally:
+        for cap in caps.values():
+            cap.release()
+        cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()

source/Helpers/HandUdpSender.py

@@ -0,0 +1,57 @@
+import socket
+import struct
+import time
+import numpy as np
+
+class HandUdpSender:
+    MAGIC = b'HAND'
+    VERSION = 2
+
+    def __init__(self, host="127.0.0.1", port=9000):
+        self.addr = (host, port)
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.seq = 0
+
+    def _encode_handedness(self, s: str) -> int:
+        if s == "Left":
+            return 1
+        if s == "Right":
+            return 2
+        return 0
+
+    def send_hands3d(self, hands3d: list[dict]):
+        """
+        hands3d: list of dicts:
+        {
+            "landmarks": (21,3) np.ndarray float,
+            "handedness": "Left"/"Right"/"Unknown",
+            "confidence": float (0..1)
+        }
+        Sends one UDP datagram containing all hands.
+        """
+        self.seq += 1
+        ts_ms = int(time.time() * 1000)
+
+        hand_count = min(len(hands3d), 255)
+
+        # Header: magic(4s), version(B), seq(I), ts(Q), hand_count(B)
+        packet = bytearray()
+        packet += struct.pack("<4sBIQB", self.MAGIC, self.VERSION, self.seq, ts_ms, hand_count)
+
+        for hid in range(hand_count):
+            h = hands3d[hid]
+            pts = np.asarray(h["landmarks"], dtype=np.float32)
+            if pts.shape != (21, 3):
+                continue
+
+            point_count = 21
+            handedness_code = self._encode_handedness(str(h.get("handedness", "Unknown")))
+            confidence = float(h.get("confidence", 0.0))
+
+            # per-hand header: hand_id(B), point_count(B), handedness(B), confidence(f)
+            packet += struct.pack("<BBBf", hid, point_count, handedness_code, confidence)
+
+            # points: 21*3 float32
+            packet += pts.tobytes(order="C")
+
+        self.sock.sendto(packet, self.addr)

source/Helpers/saveManager.py

@@ -0,0 +1,77 @@
+import json
+from pathlib import Path
+import numpy as np
+from Tbd.helper import  CameraObject
+
+def _np_to_list(a: np.ndarray):
+    return None if a is None else a.tolist()
+
+def save_calibration_json(camera_list, file_path: str, logger=None):
+    payload = {
+        "version": 1,
+        "cameras": []
+    }
+
+    for cam in camera_list:
+        payload["cameras"].append({
+            "index": int(cam.index),
+            "pxWidth": int(cam.pxWidth),
+            "pxHeight": int(cam.pxHeight),
+
+            "camera_matrix": _np_to_list(cam.camera_matrix),
+            "distortion_coefficients": _np_to_list(cam.distortion_coefficients),
+
+            "rotation_matrix_world_to_camera": _np_to_list(cam.rotation_matrix_world_to_camera),
+            "translation_vector_world_to_camera": _np_to_list(cam.translation_vector_world_to_camera),
+
+            "camera_projection_matrix": _np_to_list(cam.camera_projection_matrix),
+        })
+
+    Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(file_path, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2)
+
+    if logger:
+        logger.log(f"[Calibration] Saved calibration to {file_path}")
+
+def _list_to_np(x, shape=None):
+    if x is None:
+        return None
+    a = np.array(x, dtype=np.float64)
+    if shape is not None:
+        a = a.reshape(shape)
+    return a
+
+def load_calibration_json(file_path: str, logger=None):
+    file_path = str(file_path)
+    if not Path(file_path).exists():
+        raise FileNotFoundError(file_path)
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        payload = json.load(f)
+
+    cams = []
+    for c in payload.get("cameras", []):
+        cam = CameraObject(
+            capture=None,
+            index=int(c["index"]),
+            pxWidth=int(c.get("pxWidth", 1536)),
+            pxHeight=int(c.get("pxHeight", 2048)),
+        )
+
+        cam.camera_matrix = _list_to_np(c.get("camera_matrix"), shape=(3,3))
+        cam.distortion_coefficients = _list_to_np(c.get("distortion_coefficients"))  # keep native shape
+
+        cam.rotation_matrix_world_to_camera = _list_to_np(c.get("rotation_matrix_world_to_camera"), shape=(3,3))
+        tv = _list_to_np(c.get("translation_vector_world_to_camera"))
+        if tv is not None:
+            cam.translation_vector_world_to_camera = tv.reshape(3,1)
+
+        cam.camera_projection_matrix = _list_to_np(c.get("camera_projection_matrix"), shape=(3,4))
+        cam.load_video_capture()
+        cams.append(cam)
+
+    if logger:
+        logger.log(f"[Calibration] Loaded calibration from {file_path} ({len(cams)} cameras)")
+
+    return cams

source/Tbd/TrackingController.py

@@ -0,0 +1,260 @@
+import numpy as np
+import cv2
+from PySide6.QtCore import QObject, QTimer
+import mediapipe as mp
+from Tbd.helper import Hand2D
+from collections import defaultdict
+from Helpers.HandUdpSender import HandUdpSender
+
+import struct
+import time
+
+class TrackingController(QObject):
+    def __init__(self, cameraList, P_mats, logger=None, tick_ms=30):
+        super().__init__()
+
+        self.cameraList = cameraList
+        self.P_mats = P_mats
+        self.logger = logger
+
+        self.udp = HandUdpSender(host="127.0.0.1", port=9000)
+
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self._tick)
+        self.tick_ms = tick_ms
+
+        self.running = False
+
+        # init mediapipe here (or inject it)
+
+        self.mp_hands = mp.solutions.hands
+        self.hands = self.mp_hands.Hands(
+            static_image_mode=False,
+            max_num_hands=4,
+            model_complexity=1,
+            min_detection_confidence=0.5,
+            min_tracking_confidence=0.5,
+        )
+
+    def start(self):
+        # Guard: need projections
+        self.logger.log("Amount of P_mats:" + str(len(self.P_mats)))
+        if not self.P_mats or len(self.P_mats) < 2:
+            if self.logger: self.logger.log("[Tracking] Missing P_mats (need >=2 cameras calibrated)")
+            return
+
+        if self.running:
+            return
+        self.running = True
+        self.timer.start(self.tick_ms)
+        if self.logger: self.logger.log("[Tracking] Started")
+
+    def stop(self):
+        if not self.running:
+            return
+        self.running = False
+        self.timer.stop()
+        if self.logger: self.logger.log("[Tracking] Stopped")
+
+    def _tick(self):
+        hands2d = self._detectHands_all_cameras()
+        if not hands2d:
+            return
+
+        # Group hands across cameras (you already have _allocateDetectedHands logic)
+        hand_groups = self._allocateDetectedHands(hands2d)
+
+        # Triangulate (your existing triangulation approach)
+        hands3d = self._triangulateHands(hand_groups)
+
+        if len(hands3d) > 0:
+            self.udp.send_hands3d(hands3d)
+
+        print(
+            f"hands2d={len(hands2d)} groups={len(hand_groups)} hands3d={len(hands3d)}",
+            flush=True
+        )
+
+        if len(hands2d) > 0:
+            print("cam_ids:", sorted(set(h.camera_id for h in hands2d)), flush=True)
+
+    def _detectHands_all_cameras(self):
+        out = []
+        for camera in self.cameraList:
+            ok, frameBGR = camera.capture.read()
+            if not ok or frameBGR is None:
+                continue
+
+            h, w = frameBGR.shape[:2]
+            frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
+            frameRGB.flags.writeable = False
+
+            hand_tracking_result = self.hands.process(frameRGB)
+            if not hand_tracking_result.multi_hand_landmarks:
+                continue
+
+            handed = hand_tracking_result.multi_handedness
+            for hi, lm_list in enumerate(hand_tracking_result.multi_hand_landmarks):
+                landmarks_px = np.array(
+                    [[lm.x * w, lm.y * h] for lm in lm_list.landmark],
+                    dtype=np.float32
+                )
+
+                label, score = "Unknown", 0.0
+                if hi < len(handed) and handed[hi].classification:
+                    label = handed[hi].classification[0].label
+                    score = handed[hi].classification[0].score
+                
+                out.append(Hand2D(
+                    camera_id=camera.index,
+                    handedness=label,
+                    score=score,
+                    landmarks_px=landmarks_px
+                ))
+        return out
+
+    def _allocateDetectedHands(self, hands2d):
+        frames_by_camera = defaultdict(list)
+        for hand in hands2d:
+            frames_by_camera[hand.camera_id].append(hand)
+
+        used = set()
+        hand_groups = []
+        reproj_threshold = 50.0
+
+        camera_ids = sorted(frames_by_camera.keys())
+
+        for cam_a in camera_ids:
+            if cam_a not in self.P_mats:
+                continue
+            for hand_a in frames_by_camera[cam_a]:
+                if id(hand_a) in used:
+                    continue
+
+                group = {cam_a: hand_a}
+                used.add(id(hand_a))
+
+                P_a = self.P_mats[cam_a]
+
+                for cam_b in camera_ids:
+                    if cam_b == cam_a:
+                        continue
+                    if cam_b not in self.P_mats:
+                        continue
+
+                    P_b = self.P_mats[cam_b]
+                    best_hand = None
+                    best_err = np.inf
+
+                    for hand_b in frames_by_camera[cam_b]:
+                        if id(hand_b) in used:
+                            continue
+
+                        err = self.pair_reprojection_error(
+                            hand_a, hand_b, P_a, P_b, key_idxs=[0, 9]
+                        )
+                        if err < best_err:
+                            best_err = err
+                            best_hand = hand_b
+
+                    if best_hand is not None and best_err < reproj_threshold:
+                        group[cam_b] = best_hand
+                        used.add(id(best_hand))
+
+                hand_groups.append(group)
+
+        return hand_groups
+
+    def triangulate_hands_with_metadata(self, hand_groups):
+        hands3d = []
+        for group in hand_groups:
+            if len(group) < 2:
+                continue
+
+            handedness, confidence = self.choose_metadata_from_group(group)
+
+            cams = list(group.keys())
+            P1 = self.P_mats[cams[0]]
+            P2 = self.P_mats[cams[1]]
+            lm1 = group[cams[0]].landmarks_px
+            lm2 = group[cams[1]].landmarks_px
+
+            pts3d = []
+            for i in range(lm1.shape[0]):
+                X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
+                pts3d.append(X)
+
+            hands3d.append({
+                "handedness": handedness,          # "Left"/"Right"/"Unknown"
+                "confidence": confidence,          # 0..1
+                "landmarks": np.asarray(pts3d, dtype=np.float32)  # (21,3)
+            })
+        return hands3d
+
+    # --- math helpers (copy from your existing code) ---
+    def triangulate_point(self, P1, P2, x1, x2):
+        A = np.zeros((4, 4), dtype=np.float32)
+        A[0] = x1[0] * P1[2] - P1[0]
+        A[1] = x1[1] * P1[2] - P1[1]
+        A[2] = x2[0] * P2[2] - P2[0]
+        A[3] = x2[1] * P2[2] - P2[1]
+        _, _, Vt = np.linalg.svd(A)
+        X_h = Vt[-1]
+        X_h /= X_h[3]
+        
+        return X_h[:3]
+
+    def project_point(self, P, X):
+        X_h = np.array([X[0], X[1], X[2], 1.0], dtype=np.float32)
+        x = P @ X_h
+        return np.array([x[0] / x[2], x[1] / x[2]], dtype=np.float32)
+
+    def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs):
+        errors = []
+        for idx in key_idxs:
+            x1 = hand_a.landmarks_px[idx]
+            x2 = hand_b.landmarks_px[idx]
+            X = self.triangulate_point(P_a, P_b, x1, x2)
+            x1_hat = self.project_point(P_a, X)
+            x2_hat = self.project_point(P_b, X)
+            errors.append(np.linalg.norm(x1_hat - x1))
+            errors.append(np.linalg.norm(x2_hat - x2))
+        return float(np.mean(errors))
+
+    def choose_metadata_from_group(group: dict):
+        # group: {cam_id: Hand2D, ...}
+        best = max(group.values(), key=lambda h: float(getattr(h, "score", 0.0)))
+        handedness = getattr(best, "handedness", "Unknown")
+        confidence = float(getattr(best, "score", 0.0))
+        return handedness, confidence
+    
+    def _triangulateHands(self, hand_groups):
+        hands3d = []
+
+        for group in hand_groups:
+            cams = list(group.keys())
+            if len(cams) < 2:
+                continue
+
+            # Pick metadata from best 2D view
+            best = max(group.values(), key=lambda hh: float(getattr(hh, "score", 0.0)))
+            handedness = getattr(best, "handedness", "Unknown")
+            confidence = float(getattr(best, "score", 0.0))
+
+            P1 = self.P_mats[cams[0]]
+            P2 = self.P_mats[cams[1]]
+            lm1 = group[cams[0]].landmarks_px
+            lm2 = group[cams[1]].landmarks_px
+
+            pts3d = []
+            for i in range(lm1.shape[0]):
+                X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
+                pts3d.append(X)
+
+            hands3d.append({
+                "landmarks": np.asarray(pts3d, dtype=np.float32),
+                "handedness": handedness,
+                "confidence": confidence,
+            })
+
+        return hands3d

source/Tbd/calibration.py

@@ -0,0 +1,32 @@
+import cv2, numpy as np
+
+from helper import compute_distance
+
+cal_circle = None             # reference circle (x,y,r)
+
+def object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD):
+    global cal_circle
+    hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
+    mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255]))
+    masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask)
+    gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
+    gray = cv2.medianBlur(gray,5)
+    circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100,
+                                param1=50, param2=30, minRadius=10, maxRadius=300)
+    if circles is not None:
+        circles = np.round(circles[0]).astype(int)
+        touched = [(x,y,r) for x,y,r in circles
+                    if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD]
+        if touched:
+            touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2]))
+            x,y,r = touched[0]
+            cal_circle = (x,y,r)
+            PIXELS_PER_INCH = 2 * r
+            print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch")
+
+def draw_cal_circle(frame_vis):
+        global cal_circle
+        cx,cy,cr = cal_circle
+        cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2)
+        cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1)
+        cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)

source/Tbd/config.py

@@ -0,0 +1,30 @@
+import numpy as np
+
+# ----- Configuration -----
+CAMERA_INDICES_TO_CHECK = 10          
+FRAME_WIDTH = 3980
+FRAME_HEIGHT = 2560
+PIXELS_PER_INCH = 38          # will be set by calibration
+PINCH_THRESHOLD = 40          # px to start touch
+RELEASE_THRESHOLD = 60        # px to end touch
+CIRCLE_TOUCH_THRESHOLD = 20   # px tolerance for circle touch
+
+# HSV range for shape color (tune for your arrow: now tailored for orange)
+LOWER_SHAPE = np.array([10, 100, 100])  # hue from 10° (orange) to
+UPPER_SHAPE = np.array([30, 255, 255])  # hue up to 30°, full sat/val range
+
+DEBOUNCE_TIME   = 1.0  # seconds
+
+# ——— Constants for Tabs & Buttons ———
+TAB_W, TAB_H   = 40, 60
+TAB_X          = 600    # adjust this to your projector output width
+TAB_Y0         = 50
+TAB_GAP        = 10
+# left/right arrow buttons
+LEFT_BTN_TOP   = (TAB_X, TAB_Y0+2*(TAB_H+TAB_GAP)+20)
+RIGHT_BTN_TOP  = (TAB_X, LEFT_BTN_TOP[1]+TAB_H+TAB_GAP)
+
+DEBUG = True
+
+TARGET_FPS = 40
+FRAME_INTERVAL = 1.0 / TARGET_FPS

source/Tbd/hand_detection.py

@@ -0,0 +1,188 @@
+import mediapipe as mp
+import cv2, numpy as np, time, fitz
+from helper import compute_distance
+from config import PINCH_THRESHOLD, RELEASE_THRESHOLD
+
+# hand_state.py
+from dataclasses import dataclass, field
+from typing import Tuple, Dict, Optional
+
+from helper import finger_straight
+
+# ----- Initialize Hand Detector -----
+mp_hands = mp.solutions.hands
+mp_draw  = mp.solutions.drawing_utils
+hands = mp_hands.Hands(
+    static_image_mode=False,
+    max_num_hands=8,
+    min_detection_confidence=0.7,
+    min_tracking_confidence=0.5
+)
+
+start_pt = None               # measurement start point
+measuring = False             # measurement in progress
+
+@dataclass
+class HandState:
+    id: int                              # a persistent identifier for this hand
+    landmarks: Dict[str, Tuple[int,int]] = field(default_factory=dict)
+    finger_straightness: Dict[str, float] = field(default_factory=dict)
+    gesture: Optional[str] = None        # e.g. "pinch", "fist", "open"
+    gesture_persistence: int = 0         # how many frames the current gesture has held
+
+#handList = Dict[int, HandState] = {}
+next_id = 0 
+
+def update(mp_results, frame_vis):
+    h, w, _ = frame_vis.shape
+    detected_centroids = []
+    landmarks_list     = []
+
+    # 1) pull out centroids & raw landmarks
+    if mp_results.multi_hand_landmarks:
+        for hand in mp_results.multi_hand_landmarks:
+            pts = []
+            for lm in hand.landmark:
+                pts.append((int(lm.x*w), int(lm.y*h)))
+            centroid = np.mean(pts, axis=0)
+            detected_centroids.append(tuple(centroid.astype(int)))
+            landmarks_list.append((hand, pts))
+
+    # 2) match to existing by nearest centroid
+    new_hands = {}
+    used_ids  = set()
+    for (hand, pts), centroid in zip(landmarks_list, detected_centroids):
+        # find best existing hand
+        best_id, best_dist = None, 1e9
+        for hid, state in handList.items():
+            dx, dy = np.array(state.landmarks['centroid']) - centroid
+            d = np.hypot(dx, dy)
+            if d < best_dist and d < 100:  # 100px max match distance
+                best_dist, best_id = d, hid
+
+        if best_id is None:
+            hid = next_id
+            next_id += 1
+            state = HandState(id=hid)
+        else:
+            hid   = best_id
+            state = handList[hid]
+
+        used_ids.add(hid)
+
+        # 3) update state
+        state.persistence += 1
+        state.landmarks['centroid'] = centroid
+
+        # compute fingertip positions
+        idx_tip = pts[mp.solutions.hands.HandLandmark.INDEX_FINGER_TIP]
+        mid_tip = pts[mp.solutions.hands.HandLandmark.MIDDLE_FINGER_TIP]
+        state.landmarks['index_tip']  = idx_tip
+        state.landmarks['middle_tip'] = mid_tip
+
+        for name, tip_i, pip_i, mcp_i in [
+            ('index', mp.solutions.hands.HandLandmark.INDEX_FINGER_TIP,
+                        mp.solutions.hands.HandLandmark.INDEX_FINGER_PIP,
+                        mp.solutions.hands.HandLandmark.INDEX_FINGER_MCP),
+            ('middle', mp.solutions.hands.HandLandmark.MIDDLE_FINGER_TIP,
+                        mp.solutions.hands.HandLandmark.MIDDLE_FINGER_PIP,
+                        mp.solutions.hands.HandLandmark.MIDDLE_FINGER_MCP),
+            # add ring, pinky, thumb similarly...
+        ]:
+            p_tip = pts[tip_i]
+            p_pip = pts[pip_i]
+            p_mcp = pts[mcp_i]
+            state.finger_straightness[name] = finger_straight(p_tip, p_pip, p_mcp)
+
+        # 5) simple gesture detection
+        if state.finger_straightness['index'] < 0.2 and \
+            state.finger_straightness['middle'] < 0.2:
+            gesture = 'fist'
+        elif state.finger_straightness['index'] > 0.8 and \
+                state.finger_straightness['middle'] > 0.8:
+            gesture = 'open'
+        else:
+            gesture = None
+
+        if gesture == state.gesture:
+            state.gesture_persistence += 1
+        else:
+            state.gesture = gesture
+            state.gesture_persistence = 0
+
+        new_hands[hid] = state
+
+    # 6) drop hands not seen this frame
+    handList = new_hands
+    return list(hands.values())
+
+
+
+def detect_hands(frame):
+    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    rgb.flags.writeable = False
+    results = hands.process(rgb)    
+
+    return results
+
+
+def check_hand(hand, frame_vis, object_calibrating):
+    h, w, _ = frame_vis.shape
+
+    fingertip_idx = None
+    fingertip_mid = None
+    
+    mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS)
+    # get index and middle finger tips and PIP joints
+    idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
+    mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
+    ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
+    mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
+    fingertip_idx = (ix, iy)
+    fingertip_mid = (mx, my)
+
+    # draw fingertips
+    cv2.circle(frame_vis, fingertip_idx, 8, (255,255,0), -1)
+    cv2.circle(frame_vis, fingertip_mid,  8, (0,255,0), -1)
+
+    measuring, start_pt = is_measuring(frame_vis, hand, object_calibrating)
+
+    return fingertip_idx, measuring, start_pt
+
+
+
+def is_measuring(frame_vis, hand, object_calibrating):
+    global measuring, start_pt
+    h, w, _ = frame_vis.shape
+
+    idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
+    idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]
+    mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
+    mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP]
+
+    ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
+    mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
+    fingertip_idx = (ix, iy)
+    fingertip_mid = (mx, my)
+
+    # check extension
+    index_ext = idx_tip.y < idx_pip.y
+    middle_ext = mid_tip.y < mid_pip.y
+
+    # if measurement in progress but fingers no longer both extended, stop measuring
+    if measuring and not (index_ext and middle_ext):
+        measuring = False
+
+    # pinch distance
+    pinch = compute_distance(fingertip_idx, fingertip_mid)
+    cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2)
+
+    # measurement gesture
+    if not object_calibrating and index_ext and middle_ext:
+        if pinch < PINCH_THRESHOLD and not measuring:
+            measuring = True
+            start_pt = fingertip_idx
+        elif pinch > RELEASE_THRESHOLD and measuring:
+            measuring = False
+
+    return measuring, start_pt

source/Tbd/helper.py

@@ -0,0 +1,66 @@
+import numpy as np
+import json
+import cv2
+
+from dataclasses import dataclass, field, asdict
+from typing import List, Optional, Tuple, Dict
+
+
+# ----- Helper Functions -----
+def compute_distance(p1, p2):
+    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
+
+# 4) compute straightness per finger
+def finger_straight(i_tip, i_pip, i_mcp):
+    d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip)))
+    d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp)))
+    return float(np.clip(d1/(d2+1e-3), 0, 1))
+
+
+def default_profile():
+    return []
+
+@dataclass
+class CameraObject():
+    capture: Optional[cv2.VideoCapture]
+    index: int
+    pxHeight: int = 1536
+    pxWidth: int = 2048
+
+    # Calibration
+    camera_matrix: Optional[np.ndarray] = None          # 3x3
+    distortion_coefficients: Optional[np.ndarray] = None       # 1x5 / 1x8
+    rotation_matrix_world_to_camera: Optional[np.ndarray] = None          # 3x3 (world->cam)
+    translation_vector_world_to_camera: Optional[np.ndarray] = None          # 3x1 (world->cam)
+    camera_projection_matrix: Optional[np.ndarray] = None
+
+    def load_video_capture(self):
+        #self.capture = cv2.VideoCapture(self.index, cv2.CAP_MSMF)
+        self.capture = cv2.VideoCapture(self.index, cv2.CAP_ANY)
+        #self.capture = cv2.VideoCapture(self.index, cv2.CAP_DSHOW)
+        self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2048.0)
+        self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1536.0)
+ 
+
+@dataclass
+class Hand2D():
+    camera_id: str                   # e.g. "overhead_left"
+    handedness: str                  # "Left" / "Right"
+    score: float                     # detection/tracking confidence
+    landmarks_px: np.ndarray         # shape (21,2) in pixels
+
+
+@dataclass
+class CharucoDetection:
+    camera_id: int
+    
+    # Output of Charuco detection
+    charuco_corners: np.ndarray    # (N, 1, 2) float32
+    charuco_ids: np.ndarray        # (N, 1) int32
+
+    gray_frame: Optional[np.ndarray] = None
+
+
+@dataclass
+class CalibrationSample:
+    detections: dict[int, CharucoDetection]  # camera_id → detection

source/Tbd/import cv2.py

@@ -0,0 +1,194 @@
+
+import cv2
+import mediapipe as mp
+import numpy as np
+
+# ----- Helper Functions -----
+def compute_distance(p1, p2):
+    """Compute Euclidean distance between two points p1 and p2"""
+    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
+
+# ----- Configuration -----
+CAMERA_INDEX = 0            # Change if multiple cameras
+FRAME_WIDTH = 1280
+FRAME_HEIGHT = 720
+# Initial calibration: approximate pixels per inch
+PIXELS_PER_INCH = 20
+# Gesture thresholds
+PINCH_THRESHOLD = 40        # px distance index-middle to start action
+RELEASE_THRESHOLD = 60      # px distance to end action
+# Circle touch threshold for object calibration
+CIRCLE_TOUCH_THRESHOLD = 20  # px tolerance to detect finger on circle
+
+# ----- Initialize Hand Detector -----
+mp_hands = mp.solutions.hands
+mp_draw  = mp.solutions.drawing_utils
+hands = mp_hands.Hands(
+    static_image_mode=False,
+    max_num_hands=1,
+    min_detection_confidence=0.7,
+    min_tracking_confidence=0.5
+)
+
+# ----- State Variables -----
+measuring = False           # Flag for measurement gesture
+start_pt = None
+calibrating = False         # Flag for pinch-based calibration mode
+cal_start = None
+object_calibrating = False  # Flag for object-based calibration mode
+cal_circle = None           # Stores calibrated circle (x, y, r)
+
+# ----- Main Loop -----
+def main():
+    global measuring, start_pt, calibrating, cal_start, object_calibrating, PIXELS_PER_INCH
+
+    cap = cv2.VideoCapture(CAMERA_INDEX)
+    cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
+    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
+
+    if not cap.isOpened():
+        print(f"Error: cannot open camera {CAMERA_INDEX}")
+        return
+
+    print("Press 'c' for pinch calibration, 'o' for object circle calibration, 'q' to quit.")
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame = cv2.flip(frame, 1)
+        h, w, _ = frame.shape
+
+        # Hand detection
+        img_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        img_rgb.flags.writeable = False
+        results = hands.process(img_rgb)
+        img_rgb.flags.writeable = True
+        frame = cv2.cvtColor(img_rgb, cv2.COLOR_RGB2BGR)
+
+        fingertip_idx = None
+        fingertip_mid = None
+        index_extended = False
+        middle_extended = False
+
+        if results.multi_hand_landmarks:
+            hand = results.multi_hand_landmarks[0]
+            mp_draw.draw_landmarks(frame, hand, mp_hands.HAND_CONNECTIONS)
+            # get index and middle finger tips and PIP to check extension
+            idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
+            idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]
+            mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
+            mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP]
+            ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
+            mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
+            fingertip_idx = (ix, iy)
+            fingertip_mid = (mx, my)
+            # draw fingertips
+            cv2.circle(frame, fingertip_idx, 8, (0,255,0), -1)
+            cv2.circle(frame, fingertip_mid,  8, (0,255,0), -1)
+            # determine if fingers are extended (tip above PIP)
+            index_extended = idx_tip.y < idx_pip.y
+            middle_extended = mid_tip.y < mid_pip.y
+            # pinch distance between index and middle
+            pinch_dist = compute_distance(fingertip_idx, fingertip_mid)
+            cv2.putText(frame, f"Pinch: {int(pinch_dist)}px", (10,30),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0), 2)
+
+            # Pinch-based calibration
+            if calibrating:
+                if pinch_dist < PINCH_THRESHOLD and cal_start is None:
+                    cal_start = fingertip_idx
+                    print("Pinch calibration start point set.")
+                elif pinch_dist > RELEASE_THRESHOLD and cal_start is not None:
+                    cal_end = fingertip_idx
+                    px_dist = compute_distance(cal_start, cal_end)
+                    inches = float(input("Enter actual distance between points in inches: "))
+                    PIXELS_PER_INCH = px_dist / inches
+                    print(f"Pinch calibration done: {PIXELS_PER_INCH:.2f} pixels/inch")
+                    calibrating = False
+                    cal_start = None
+
+            # Measurement gesture (only when not calibrating)
+            elif not object_calibrating and index_extended and middle_extended:
+                if pinch_dist < PINCH_THRESHOLD and not measuring:
+                    measuring = True
+                    start_pt = fingertip_idx
+                elif pinch_dist > RELEASE_THRESHOLD and measuring:
+                    measuring = False
+
+        # Object-based calibration
+        if object_calibrating:
+            # Mask for orange color to find printed reference circle
+            hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
+            # HSV range for orange (tune as needed)
+            lower_orange = np.array([10, 100, 100])
+            upper_orange = np.array([25, 255, 255])
+            color_mask = cv2.inRange(hsv, lower_orange, upper_orange)
+            masked_frame = cv2.bitwise_and(frame, frame, mask=color_mask)
+            # Convert masked area to grayscale for Hough
+            gray = cv2.cvtColor(masked_frame, cv2.COLOR_BGR2GRAY)
+            gray = cv2.medianBlur(gray, 5)
+            circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, dp=1.2, minDist=100,
+                                       param1=50, param2=30, minRadius=10, maxRadius=300)
+            if circles is not None and fingertip_idx is not None:
+                circles = np.round(circles[0, :]).astype(int)
+                # Filter circles by proximity of fingertip to circumference
+                touched = []
+                for x, y, r in circles:
+                    dist_c = compute_distance((x, y), fingertip_idx)
+                    if abs(dist_c - r) < CIRCLE_TOUCH_THRESHOLD:
+                        touched.append((x, y, r))
+                if touched:
+                    # choose circle closest to exact touch point
+                    touched.sort(key=lambda c: abs(compute_distance((c[0], c[1]), fingertip_idx) - c[2]))
+                    x, y, r = touched[0]
+                    # store calibrated circle permanently
+                    cal_circle = (x, y, r)
+                    # draw selected calibration circle
+                    cv2.circle(frame, (x, y), r, (0, 0, 255), 3)
+                    cv2.drawMarker(frame, (x, y), (0, 0, 255), markerType=cv2.MARKER_CROSS, markerSize=20, thickness=2)
+                    cv2.line(frame, (x - r, y), (x + r, y), (0, 0, 255), 2)
+                    cv2.putText(frame, f"Cal Circle r={r}px", (x - r, y - r - 10),
+                                cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,0,255), 2)
+                    # compute pixels per inch from diameter
+                    PIXELS_PER_INCH = (2 * r) / 1.0
+                    print(f"Object calibration done: {PIXELS_PER_INCH:.2f} pixels/inch")
+                    object_calibrating = False
+
+        # Overlay mode text
+        if calibrating:
+            cv2.putText(frame, "PINCH CALIBRATING...", (10,60), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
+        if object_calibrating:
+            cv2.putText(frame, 'PLACE 1" CIRCLE & POINT AT IT', (10,90),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0,0,255), 2)
+
+        # Draw measurement line and values
+        if measuring and start_pt and fingertip_idx:
+            cv2.line(frame, start_pt, fingertip_idx, (255,0,0), 2)
+            px = compute_distance(start_pt, fingertip_idx)
+            inch = px / PIXELS_PER_INCH
+            cm = inch * 2.54
+            midpt = ((start_pt[0] + fingertip_idx[0])//2,
+                     (start_pt[1] + fingertip_idx[1])//2)
+            cv2.putText(frame, f"{inch:.2f} in / {cm:.1f} cm", (midpt[0]+10, midpt[1]-10),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,0,0), 2)
+
+        # Display
+        cv2.imshow('Hand Measure', frame)
+        key = cv2.waitKey(1) & 0xFF
+        if key == ord('q'):
+            break
+        elif key == ord('c'):
+            calibrating = True
+            cal_start = None
+            print("Entered pinch calibration mode.")
+        elif key == ord('o'):
+            object_calibrating = True
+            print("Entered object calibration mode. Present a 1-inch circle & point at it.")
+
+    cap.release()
+    cv2.destroyAllWindows()
+
+if __name__ == '__main__':
+    main()

source/UI/CameraSetupWidget.py

@@ -0,0 +1,67 @@
+import cv2, numpy as np
+
+from Tbd.helper import  CameraObject
+
+from PySide6.QtCore import Qt, QTimer, QSize
+from PySide6.QtGui import QImage, QPixmap
+
+from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QSizePolicy, QHBoxLayout, QLineEdit, QPushButton
+from typing import List
+from PySide6.QtWidgets import QScrollArea, QFrame
+
+
+class CameraSetupWidget(QWidget):
+
+    def __init__(self, cameraObject: CameraObject):
+        super().__init__()
+        
+        self.cameraObject = cameraObject
+        
+        self.mainLayout = QHBoxLayout(self)
+
+        self.frame_preview_widget = QLabel()
+        self.preview_max_size = QSize(800, 600)
+        self.frame_preview_widget.setMaximumSize(self.preview_max_size)
+        self.camerOptionsLayout = QVBoxLayout()
+
+        self.start_preview_button = QPushButton("Start")
+        self.stop_preview_button = QPushButton("Stop")
+
+
+
+        self.camerOptionsLayout.addWidget(self.start_preview_button)
+        self.camerOptionsLayout.addWidget(self.stop_preview_button)
+
+        self.mainLayout.addLayout(self.camerOptionsLayout)
+        self.mainLayout.addWidget(self.frame_preview_widget)
+
+        self.start_preview_button.clicked.connect(self._start_camera_preview)
+        self.stop_preview_button.clicked.connect(self._stop_camera_preview)
+
+        self.frame_update_timer = QTimer(self)
+        self.frame_update_timer.timeout.connect(self._update_frame)
+        
+
+    def _start_camera_preview(self):
+        self.frame_update_timer.start(30)
+        
+    def _stop_camera_preview(self):
+        self.frame_update_timer.stop()
+    
+
+    
+    def _update_frame(self):
+        ok, frame = self.cameraObject.capture.read()
+        if not ok or frame is None:
+            return
+        
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        height, width, ch = frame_rgb.shape
+        qimg = QImage(frame_rgb.data, width, height, ch * width, QImage.Format_RGB888)
+        pixmap = QPixmap.fromImage(qimg)
+        pixmap = pixmap.scaled(
+            self.frame_preview_widget.maximumSize(),
+            Qt.KeepAspectRatio,
+            Qt.SmoothTransformation,
+        )
+        self.frame_preview_widget.setPixmap(pixmap)

source/UI/GameWindow.py

@@ -0,0 +1,333 @@
+import cv2
+import numpy
+import mediapipe
+from Tbd.helper import CameraObject, Hand2D
+from PySide6.QtCore import Qt, QTimer
+from PySide6.QtWidgets import QMainWindow, QLabel, QVBoxLayout, QWidget
+from PySide6.QtGui import QImage, QPixmap
+from typing import List
+from collections import defaultdict
+import winsound
+
+mediapipe_options = mediapipe.solutions.hands
+
+class GameWindow(QMainWindow):
+    """
+    Separate window for the “game” part.
+    You can put your rendering, controls, etc. here.
+    """
+
+    def __init__(self, cameraList: List[CameraObject], width_px=1920, height_px=1080, fps=30, max_num_hands=4):
+        super().__init__()
+        
+        self.P_mats = {}
+
+        self.cameraList = cameraList
+        self._hands = mediapipe_options.Hands(
+            static_image_mode=False,
+            max_num_hands=max_num_hands,
+            model_complexity=1,
+            min_detection_confidence=0.5,
+            min_tracking_confidence=0.5,
+        )
+
+        self.setWindowTitle("Game Window")
+        self.canvas_w = width_px
+        self.canvas_h = height_px
+
+
+        self.view = QLabel("HUD")
+        self.view.setAlignment(Qt.AlignCenter)
+        self.view.setScaledContents(False)  # keep aspect ratio
+
+
+        central = QWidget(self)
+        self.setCentralWidget(central)
+        layout = QVBoxLayout(central)
+        layout.setContentsMargins(0, 0, 0, 0)
+        layout.addWidget(self.view, 1)
+  
+        # Create a Timer for my _tick function and start it
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self._tick)
+        self.timer.start(30)  # ~33 FPS
+
+    def _tick(self):
+
+        # 1) create a black BGR canvas
+        frame_bgr = numpy.zeros((self.canvas_h, self.canvas_w, 3), dtype=numpy.uint8)
+
+        # 2) (optional) draw HUD widgets here
+        cv2.putText(frame_bgr, "HUD ready", (40, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (200,200,200), 2, cv2.LINE_AA)
+
+        hands2d = self._detectHands()
+        
+        self.draw_hands_on_frame(frame_bgr, hands2d)
+
+        # 3) draw detected hands (in canvas pixel coords)
+        for h in hands2d:
+            for (x, y) in h.landmarks_px.astype(int):
+                cv2.circle(frame_bgr, (x, y), 4, (0, 255, 255), -1, cv2.LINE_AA)
+
+        # 4) show it
+        frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB)
+        qimg = QImage(frame_rgb.data, self.canvas_w, self.canvas_h,
+                      self.canvas_w * 3, QImage.Format_RGB888)
+        pix = QPixmap.fromImage(qimg)
+        # keep aspect ratio when fitting into the label
+        self.view.setPixmap(pix.scaled(self.view.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation))
+
+        hand_groups = self._allocateDetectedHands(hands2d)
+
+        hands3d = self._triangulateHands(hand_groups)
+        print(hands3d)
+        self._updateGeasture()
+        self._checkInteractions()
+        self._upateHUD()
+        self._drawHUD()
+        
+
+    # Loop all Cameras and return a List of all found Hands in all Cameras
+    def _detectHands(self):
+        out: List[Hand2D] = [] # List of all found Hands
+
+
+        for camera in self.cameraList:
+
+            # Get the frame of the camera and check if the camera is available
+            ok, frameBGR = camera.capture.read()
+            if not ok or frameBGR is None:
+                continue
+
+            frameHight, frameWidht = frameBGR.shape[:2]             # Take only the the frame hight and widht from the Tuple
+            
+            frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)    # Convert to RGB becaus,e MediaPipe expects RGB
+            frameRGB.flags.writeable = False                        # (minor speed gain)
+            
+            # Detect Hands in the frame and return a list of all found Hands, then check if 
+            foundHands = self._hands.process(frameRGB)
+            if not foundHands.multi_hand_landmarks:
+                continue
+
+            # handedness info aligns with landmarks list
+            handed = foundHands.multi_handedness
+
+            for handIndex, landmarlList in enumerate(foundHands.multi_hand_landmarks):
+                landmarks_px = numpy.array([[landmark.x*frameWidht, landmark.y*frameHight] for landmark in landmarlList.landmark], dtype=numpy.float32)
+                
+                label = "Unknown"
+                score = 0.0
+
+                if handIndex < len(handed):
+                    classifications = handed[handIndex].classification
+                    if classifications:
+                        label = classifications[0].label  # "Left" / "Right"
+                        score = classifications[0].score  # confidence
+
+                out.append(Hand2D(
+                    camera_id=camera.index, 
+                    handedness=label, 
+                    score=score,
+                    landmarks_px=landmarks_px
+                ))
+                               
+
+        return out
+
+    def _allocateDetectedHands(self, hands2d: list):
+        """
+        Take flat list of Hand2D from all cameras and group them into physical hands.
+        Writes self._hand_groups = [ {cam_id: Hand2D, ...}, ... ]
+        """
+   
+        # Group by camera
+        frames_by_camera = defaultdict(list)
+        for hand in hands2d:
+            frames_by_camera[hand.camera_id].append(hand)
+
+        # We will mark which Hand2D detections are already consumed
+        used = set()  # set of id(hand2d)
+        hand_groups = []
+
+        # Threshold in pixels for “same hand”
+        reproj_threshold = 8.0  # tune this
+
+        camera_ids = sorted(frames_by_camera.keys())
+
+        for cam_a in camera_ids:
+            for hand_a in frames_by_camera[cam_a]:
+                if id(hand_a) in used:
+                    continue
+
+                # Start a new group with this detection as the seed
+                group = {cam_a: hand_a}
+                used.add(id(hand_a))
+
+                P_a = self.P_mats[cam_a]
+
+                # Try to find matching hands in all other cameras
+                for cam_b in camera_ids:
+                    if cam_b == cam_a:
+                        continue
+                    if cam_b not in self.P_mats:
+                        continue
+
+                    P_b = self.P_mats[cam_b]
+
+                    best_hand = None
+                    best_err = numpy.inf
+
+                    for hand_b in frames_by_camera[cam_b]:
+                        if id(hand_b) in used:
+                            continue
+
+                        err = self.pair_reprojection_error(
+                            hand_a, hand_b,
+                            P_a, P_b,
+                            key_idxs=[0, 9]  # wrist + middle MCP for example
+                        )
+
+                        if err < best_err:
+                            best_err = err
+                            best_hand = hand_b
+
+                    if best_hand is not None and best_err < reproj_threshold:
+                        group[cam_b] = best_hand
+                        used.add(id(best_hand))
+
+                hand_groups.append(group)
+
+        return hand_groups
+        
+    
+    def _triangulateHands(self, hand_groups):
+
+        hands3d = []
+
+        for group in hand_groups:
+            # use all cameras in 'group' to triangulate each landmark
+            cams = list(group.keys())
+            hands = [group[c] for c in cams]
+
+            # Example: simple pairwise triangulation using first two cameras
+            if len(cams) < 2:
+                continue  # need at least 2 views
+
+            P1 = self.P_mats[cams[0]]
+            P2 = self.P_mats[cams[1]]
+            lm1 = hands[0].landmarks_px
+            lm2 = hands[1].landmarks_px
+
+            pts3d = []
+            for i in range(lm1.shape[0]):
+                X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
+                pts3d.append(X)
+
+            pts3d = numpy.array(pts3d, dtype=numpy.float32)
+            hands3d.append(pts3d)
+
+        return hands3d
+
+    def _updateGeasture(self):    
+
+        return
+    
+    def _getMostConfindentHand(self):
+
+        return
+
+    def _checkInteractions(self):
+
+        return
+    
+    def _upateHUD(self):
+
+        return
+
+    def _drawHUD(self):
+
+        return
+    
+
+    def close(self):
+        print("What?")
+        self.timer.stop()
+        return super().close()
+    
+
+    def draw_hands_on_frame(self, frame_bgr: numpy.ndarray, hands2d: list):
+        # Draw connections first, then points
+        for h in hands2d:
+            pts = h.landmarks_px.astype(int)  # (21,2) in pixels
+
+            # bones
+            for a, b in mediapipe_options.HAND_CONNECTIONS:
+                cv2.line(frame_bgr,
+                        (int(pts[a,0]), int(pts[a,1])),
+                        (int(pts[b,0]), int(pts[b,1])),
+                        (0, 255, 0), 2, cv2.LINE_AA)
+
+            # joints
+            for (x, y) in pts:
+                cv2.circle(frame_bgr, (int(x), int(y)), 3, (0, 0, 255), -1, cv2.LINE_AA)
+
+            # optional label
+            cv2.putText(frame_bgr, f"{h.handedness} {h.score:.2f}",
+                        (int(pts[0,0]), int(pts[0,1])-8),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (80,255,80), 1, cv2.LINE_AA)
+        
+    def project_point(self, P, X):
+        """
+        P: 3x4 projection matrix
+        X: 3D point (X, Y, Z)
+        returns: 2D point (u, v) in pixels
+        """
+        X_h = numpy.array([X[0], X[1], X[2], 1.0], dtype=numpy.float32)
+        x = P @ X_h
+        return numpy.array([x[0] / x[2], x[1] / x[2]], dtype=numpy.float32)
+    
+    def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs):
+        """
+        hand_a, hand_b: Hand2D objects
+        P_a, P_b: 3x4 projection matrices
+        key_idxs: list of landmark indices to use (e.g. [0, 9]) 
+        returns: average reprojection error in pixels
+        """
+        errors = []
+
+        for idx in key_idxs:
+            x1 = hand_a.landmarks_px[idx]  # (u, v)
+            x2 = hand_b.landmarks_px[idx]
+
+            X = self.triangulate_point(P_a, P_b, x1, x2)
+
+            x1_hat = self.project_point(P_a, X)
+            x2_hat = self.project_point(P_b, X)
+
+            e1 = numpy.linalg.norm(x1_hat - x1)
+            e2 = numpy.linalg.norm(x2_hat - x2)
+
+            errors.append(e1)
+            errors.append(e2)
+
+        return float(numpy.mean(errors))
+
+    
+    def triangulate_point(self, P1, P2, x1, x2):
+        """
+        P1, P2: 3x4 projection matrices
+        x1, x2: 2D points (u, v) in pixels (float)
+        returns: 3D point in world coords (X, Y, Z)
+        """
+        A = numpy.zeros((4, 4), dtype=numpy.float32)
+        A[0] = x1[0] * P1[2] - P1[0]
+        A[1] = x1[1] * P1[2] - P1[1]
+        A[2] = x2[0] * P2[2] - P2[0]
+        A[3] = x2[1] * P2[2] - P2[1]
+
+        # Solve A * X = 0, X is homogeneous 4D
+        _, _, Vt = numpy.linalg.svd(A)
+        X_h = Vt[-1]
+        X_h /= X_h[3]
+        return X_h[:3]  # (X, Y, Z)
+

source/UI/HomePageWidget.py

@@ -0,0 +1,505 @@
+import cv2
+import numpy
+
+from Tbd.helper import  CameraObject, CharucoDetection, CalibrationSample
+from UI.CameraSetupWidget import CameraSetupWidget
+from Helpers import saveManager as sm
+
+from PySide6.QtCore import Qt, QTimer
+from PySide6.QtGui import QImage, QPixmap
+
+from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QSizePolicy
+from typing import List
+from PySide6.QtWidgets import QScrollArea, QFrame, QPushButton
+
+from UI.UILogger import UILogger
+
+MIN_CHARUCO_CORNERS = 15   # absolute minimum
+#GOOD_CHARUCO_CORNERS = 25  # ideal
+
+DICT_ID = cv2.aruco.DICT_4X4_50
+SQUARES_X = 6                    # number of chessboard squares in X
+SQUARES_Y = 9                    # number of chessboard squares in Y
+SQUARE_LEN_MM = 25.0             # square size in mm
+MARKER_LEN_MM = 18.0             # marker size in mm (must be < square)
+
+#FRAME_SIZE = [3264, 2448]
+#FRAME_SIZE = [1536, 2048]
+FRAME_SIZE = [2048, 1536]
+
+REFERENCE_CAMERA_INDEX = 1
+
+class HomeWindow(QWidget):
+
+    def __init__(self, cameraList: List[CameraObject], on_cameraList_changed, P_mats, logger : UILogger):
+        super().__init__()
+
+        # Initiating variables
+        self.P_mats = P_mats
+        self.on_cameraList_changed = on_cameraList_changed
+        self.cameraList = cameraList
+        self.logger = logger
+        self.calibration_samples: list[CalibrationSample] = []
+        self.camera_Setup_Widget_List = []
+
+
+        # Charcuo detection stuff
+        dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID)
+        self.board = cv2.aruco.CharucoBoard(
+            (SQUARES_X, SQUARES_Y),
+            SQUARE_LEN_MM,
+            MARKER_LEN_MM,
+            dictionary
+        )
+        self.charuco_detector = cv2.aruco.CharucoDetector(self.board)
+        # Detector parameters
+        detector_params = cv2.aruco.DetectorParameters()
+            # You can tweak these if detection is unstable:
+            # detector_params.adaptiveThreshWinSizeMin = 3
+            # detector_params.adaptiveThreshWinSizeMax = 23
+            # detector_params.adaptiveThreshWinSizeStep = 10
+
+        self.arcuo_detector = cv2.aruco.ArucoDetector(dictionary, detector_params)
+
+        # Creating the main_Layout and other Widgets 
+        self.main_Layout = QVBoxLayout(self)
+        self.scroll_Box = QScrollArea()
+        self.calibration_button = QPushButton("Calibrate")
+        self.caputre_calibration_sample_button = QPushButton("Capture calibration sample")
+        self.clear_calibration_samples_button = QPushButton("Clear calibration samples")
+        self.load_calibration_button = QPushButton("Load calibration")
+        self.save_calibration_button = QPushButton("Save calibration")
+        
+
+        # Creating the Scrollable area
+        self.scroll_Box.setWidgetResizable(True)
+        self.scroll_Content = QWidget()
+        self.scroll_Layout = QVBoxLayout(self.scroll_Content)
+        self.scroll_Layout.setAlignment(Qt.AlignTop)
+        self.scroll_Box.setWidget(self.scroll_Content)
+
+        # Adding Primary show elements
+        self.main_Layout.addWidget(self.clear_calibration_samples_button)
+        self.main_Layout.addWidget(self.load_calibration_button)
+        self.main_Layout.addWidget(self.save_calibration_button)
+        self.main_Layout.addWidget(self.caputre_calibration_sample_button)
+        self.main_Layout.addWidget(self.calibration_button)
+        self.main_Layout.addWidget(self.scroll_Box)
+        
+        # Connecting Buttons to functions
+        self.calibration_button.clicked.connect(self._calibrate_camera)
+        self.caputre_calibration_sample_button.clicked.connect(self._caputre_calibration_sample)
+        self.clear_calibration_samples_button.clicked.connect(self._clear_calibration_samples)
+        self.load_calibration_button.clicked.connect(self._load_calibration)
+        self.save_calibration_button.clicked.connect(self._save_calibration)
+
+
+    def _clear_calibration_samples(self):
+        self.calibration_samples.clear()
+        self.logger.log("Calibration samples where cleared.")
+
+    def _load_calibration(self):
+        loaded = sm.load_calibration_json("C:\\git\\Table\\Test\\test.json", self.logger)
+        self.cameraList.clear()
+        self.cameraList.extend(loaded)
+        self.P_mats.clear()
+        self.P_mats.update({
+            camera.index: camera.camera_projection_matrix
+            for camera in self.cameraList
+            if camera.camera_projection_matrix is not None
+        })
+        self.on_cameraList_changed()
+
+    def _save_calibration(self):
+        sm.save_calibration_json(self.cameraList, "C:\\git\\Table\\Test\\test.json", self.logger) 
+
+    def _calibrate_camera(self):
+        self.logger.log("Fuck you")
+
+        if(len(self.cameraList) < 2):
+            self.logger.log("You need at least 2 cameras for calibration")
+            return
+
+        self._clear_camera_calibrations()
+
+        REFERENCE_CAMERA_INDEX = self.cameraList[0].index
+
+        for camera in self.cameraList:
+            reprojection_error_rms, camera_matrix, distortion_coefficients = self._calibrate_intrinsics_characuo_for_camera(
+                sample_list = self.calibration_samples,
+                camera_index = camera.index,
+                board = self.board
+            )
+
+            camera.camera_matrix = camera_matrix
+            camera.distortion_coefficients = distortion_coefficients
+
+            self.logger.log(f"[Intrinsics] cam{camera.index}: RMS={reprojection_error_rms:.4f} fx={camera_matrix[0,0]:.2f} fy={camera_matrix[1,1]:.2f} cx={camera_matrix[0,2]:.2f} cy={camera_matrix[1,2]:.2f}")
+
+        reference_camera = next(camera for camera in self.cameraList if camera.index == REFERENCE_CAMERA_INDEX)
+
+        self.logger.log("Intrinsics done")
+        
+        self.logger.log("Start extrinsics1")
+        # Reference camera projection
+        reference_camera.rotation_matrix_world_to_camera = numpy.eye(3)
+        reference_camera.translation_vector_world_to_camera = numpy.zeros((3,1))
+        reference_camera.camera_projection_matrix = (
+            reference_camera.camera_matrix
+            @ numpy.hstack([numpy.eye(3), numpy.zeros((3,1))])
+        )
+        self.logger.log("Start extrinsics2")
+        self.P_mats.clear()
+        self.P_mats[reference_camera.index] = reference_camera.camera_projection_matrix
+        self.logger.log("Start extrinsics3")
+        for camera_to_calibrate in self.cameraList:
+            if camera_to_calibrate.index == REFERENCE_CAMERA_INDEX:
+                continue
+
+             
+            stereo_rms,R_ref_to_cam,t_ref_to_cam,E, F, P_ref, P_cam = self._stereo_calibrate_from_charuco_samples(
+                sample_list=self.calibration_samples,
+                board=self.board,
+                camera_refernece=reference_camera,
+                camera_to_calibrate=camera_to_calibrate
+            ) 
+            
+
+            camera_to_calibrate.rotation_matrix_world_to_camera = R_ref_to_cam
+            camera_to_calibrate.translation_vector_world_to_camera = t_ref_to_cam
+            camera_to_calibrate.camera_projection_matrix = P_cam
+
+            self.P_mats[camera_to_calibrate.index] = camera_to_calibrate.camera_projection_matrix
+
+        
+        self.logger.log("Extrinsics done")
+        self.logger.log("Amount of P_mats:" + str(len(self.P_mats)))
+        self.logger.log("Amount of cameras:" + str(len(self.cameraList)))
+        self.log_all_camera_calibration(self.cameraList)
+
+
+    def rotation_matrix_to_rpy_deg(self, R: numpy.ndarray):
+        """
+        Convert rotation matrix to roll, pitch, yaw in degrees.
+        Assumes right-handed, OpenCV convention.
+        """
+        sy = numpy.sqrt(R[0,0]*R[0,0] + R[1,0]*R[1,0])
+        singular = sy < 1e-6
+
+        if not singular:
+            roll  = numpy.arctan2(R[2,1], R[2,2])
+            pitch = numpy.arctan2(-R[2,0], sy)
+            yaw   = numpy.arctan2(R[1,0], R[0,0])
+        else:
+            roll  = numpy.arctan2(-R[1,2], R[1,1])
+            pitch = numpy.arctan2(-R[2,0], sy)
+            yaw   = 0.0
+
+        return numpy.degrees([roll, pitch, yaw])
+    
+    def log_all_camera_calibration(self, camera_list):
+        self.logger.log("========== CAMERA CALIBRATION SUMMARY ==========")
+
+        for cam in camera_list:
+            self.logger.log(f"--- Camera {cam.index} ---")
+
+            # -------- Intrinsics --------
+            if cam.camera_matrix is not None:
+                K = cam.camera_matrix
+                fx, fy = K[0,0], K[1,1]
+                cx, cy = K[0,2], K[1,2]
+
+                self.logger.log(
+                    f"Intrinsics:"
+                    f" fx={fx:.2f}, fy={fy:.2f},"
+                    f" cx={cx:.2f}, cy={cy:.2f}"
+                )
+
+                if cam.distortion_coefficients is not None:
+                    d = cam.distortion_coefficients.flatten()
+                    d_short = ", ".join(f"{v:.4f}" for v in d[:5])
+                    self.logger.log(f"Distortion: [{d_short}{'...' if len(d) > 5 else ''}]")
+            else:
+                self.logger.log("Intrinsics: NOT SET")
+
+            # -------- Extrinsics --------
+            if cam.rotation_matrix_world_to_camera is not None and cam.translation_vector_world_to_camera is not None:
+                R = cam.rotation_matrix_world_to_camera
+                t = cam.translation_vector_world_to_camera.reshape(3)
+
+                roll, pitch, yaw = self.rotation_matrix_to_rpy_deg(R)
+                dist = numpy.linalg.norm(t)
+
+                self.logger.log(
+                    f"Extrinsics (world → cam):"
+                    f" t=({t[0]:.1f}, {t[1]:.1f}, {t[2]:.1f})"
+                    f" | |t|={dist:.1f}"
+                )
+                self.logger.log(
+                    f"Rotation (deg):"
+                    f" roll={roll:.2f}, pitch={pitch:.2f}, yaw={yaw:.2f}"
+                )
+            else:
+                self.logger.log("Extrinsics: NOT SET")
+
+            # -------- Projection --------
+            if cam.camera_projection_matrix is not None:
+                P = cam.camera_projection_matrix
+                self.logger.log(f"Projection matrix: shape={P.shape}")
+            else:
+                self.logger.log("Projection matrix: NOT SET")
+
+        self.logger.log("==============================================")
+
+    def _build_stereo_correspondences_from_samples(
+            self,
+            sample_list,
+            board,
+            camera_index_reference,
+            camera_index_to_calibrate
+    ):
+        object_points_per_frame  = []
+        frame_points_per_frame_reference  = []
+        frame_points_per_frame_to_calibrate  = []
+
+        for sample in sample_list:
+
+            # Get detections for both cameras and check if valide
+            detection_reference = sample.detections.get(camera_index_reference)
+            detection_to_calibrate = sample.detections.get(camera_index_to_calibrate)
+
+            if detection_reference is None or detection_to_calibrate is None:
+                continue
+
+            if detection_reference.charuco_ids is None or detection_reference.charuco_corners is None:
+                continue
+
+            if detection_to_calibrate.charuco_ids is None or detection_to_calibrate.charuco_corners is None:
+                continue
+
+            # Find corner IDs that both cameras can see and check if the amount is enough
+            ids_reference = detection_reference.charuco_ids.reshape(-1)
+            ids_to_calibrate = detection_to_calibrate.charuco_ids.reshape(-1)
+
+            common_ids = numpy.intersect1d(ids_reference, ids_to_calibrate)
+
+            if len(common_ids) < MIN_CHARUCO_CORNERS:
+                continue
+
+            # Build ordered correspondences by common_ids
+            # Map id -> corner for each cam
+            map_a = {int(i): detection_reference.charuco_corners[idx] for idx, i in enumerate(ids_reference)}
+            map_b = {int(i): detection_to_calibrate.charuco_corners[idx] for idx, i in enumerate(ids_to_calibrate)}
+
+            # Assemble corners/ids arrays in matching order
+            corners_reference = numpy.array([map_a[int(i)] for i in common_ids], dtype=numpy.float32).reshape(-1, 1, 2)
+            corners_to_calibrate = numpy.array([map_b[int(i)] for i in common_ids], dtype=numpy.float32).reshape(-1, 1, 2)
+            ids_common = common_ids.astype(numpy.int32).reshape(-1, 1)
+
+             # Convert ChArUco corners+ids -> (objectPts, imagePts) for the board
+            object_points_reference, frame_pts_reference = board.matchImagePoints(corners_reference, ids_common)
+            _, frame_pts_to_calibrate = board.matchImagePoints(corners_to_calibrate, ids_common)
+
+            if object_points_reference is None or frame_pts_reference is None or frame_pts_to_calibrate is None:
+                continue
+
+            object_points_reference = self._as_np_float32(object_points_reference)
+            frame_pts_reference = self._as_np_float32(frame_pts_reference)
+            frame_pts_to_calibrate = self._as_np_float32(frame_pts_to_calibrate)
+
+            if len(object_points_reference) < MIN_CHARUCO_CORNERS:
+                continue
+
+            object_points_per_frame.append(object_points_reference)
+            frame_points_per_frame_reference.append(frame_pts_reference)
+            frame_points_per_frame_to_calibrate.append(frame_pts_to_calibrate)
+            
+        return object_points_per_frame, frame_points_per_frame_reference, frame_points_per_frame_to_calibrate
+
+    def _as_np_float32(self, x):
+        return numpy.asarray(x, dtype=numpy.float32)
+
+    def _stereo_calibrate_from_charuco_samples(
+            self,
+            sample_list,
+            board,
+            camera_refernece,
+            camera_to_calibrate
+    ):
+        
+        if camera_refernece.camera_matrix is None or camera_refernece.distortion_coefficients is None:
+            self.logger.log(f"cam{camera_refernece.index} missing intrinsics")
+        if camera_to_calibrate.camera_matrix is None or camera_to_calibrate.distortion_coefficients is None:
+            self.logger.log(f"cam{camera_to_calibrate.index} missing intrinsics")
+
+        object_points_per_frame, frame_points_per_frame_reference, frame_points_per_frame_to_calibrate = self._build_stereo_correspondences_from_samples(
+            sample_list=sample_list,
+            board=board,
+            camera_index_reference=camera_refernece.index,
+            camera_index_to_calibrate=camera_to_calibrate.index
+        )
+
+        frame_width, frame_height = FRAME_SIZE
+
+        # Keep intrinsics fixed (recommended since you already calibrated them)
+        flags = cv2.CALIB_FIX_INTRINSIC
+
+        # Termination criteria for stereo calibration optimizer
+        criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 1e-6)
+
+        stereo_rms_reprojection_error_px, camera_matrix_reference, dist_reference, camera_matrix_to_calibrate, dist_to_calibrate, \
+            rotation_matrix_reference_to_calibrate, translation_vector_reference_to_calibrate, \
+            essential_matrix, fundamental_matrix = cv2.stereoCalibrate(
+            objectPoints=object_points_per_frame,
+            imagePoints1=frame_points_per_frame_reference,
+            imagePoints2=frame_points_per_frame_to_calibrate,
+            cameraMatrix1=camera_refernece.camera_matrix,
+            distCoeffs1=camera_refernece.distortion_coefficients,
+            cameraMatrix2=camera_to_calibrate.camera_matrix,
+            distCoeffs2=camera_to_calibrate.distortion_coefficients,
+            imageSize=(frame_width, frame_height),
+            criteria=criteria,
+            flags=flags
+        )
+
+        projection_matrix_camA = camera_refernece.camera_matrix @ numpy.hstack([numpy.eye(3, dtype=numpy.float64), numpy.zeros((3,1), dtype=numpy.float64)])
+        projection_matrix_camB = camera_to_calibrate.camera_matrix @ numpy.hstack([rotation_matrix_reference_to_calibrate, translation_vector_reference_to_calibrate])
+
+
+
+        return (
+            float(stereo_rms_reprojection_error_px),
+            rotation_matrix_reference_to_calibrate,
+            translation_vector_reference_to_calibrate,
+            essential_matrix,
+            fundamental_matrix,
+            projection_matrix_camA,
+            projection_matrix_camB,
+        )
+
+    def _clear_camera_calibrations(self):
+        for camera in self.cameraList:
+            camera.camera_matrix = None
+            camera.distortion_coefficients = None
+            camera.rotation_matrix_world_to_camera = None
+            camera.translation_vector_world_to_camera = None
+
+    def _calibrate_intrinsics_characuo_for_camera(
+            self,
+            sample_list: CalibrationSample,
+            camera_index: int,
+            board
+    ):
+        object_points_per_frame  = []
+        frame_points_per_frame  = []
+
+        for sample in sample_list:
+            detection = sample.detections.get(camera_index)
+
+            if detection is None:
+                continue
+
+            if detection.charuco_ids is None or detection.charuco_corners is None:
+                continue
+
+            if len(detection.charuco_ids) < MIN_CHARUCO_CORNERS:
+                continue
+
+            # map detected 2D corners + ids to corresponding 3D board points
+            object_points, frame_points = board.matchImagePoints(detection.charuco_corners, detection.charuco_ids)
+
+            if object_points is None or frame_points is None:
+                continue
+
+            object_points_per_frame.append(object_points)
+            frame_points_per_frame.append(frame_points)
+
+
+        
+        frame_width, frame_height = FRAME_SIZE
+        self.logger.log(f"Calibrating cam{camera_index} with imageSize={FRAME_SIZE} (w,h)")
+        self.logger.log(f"H= {frame_width} W= {frame_height}")
+
+        reprojection_error_rms, camera_matrix, distortion_coefficients, rvecs, tvecs = cv2.calibrateCamera(
+            objectPoints=object_points_per_frame,
+            imagePoints=frame_points_per_frame,
+            imageSize=(frame_width, frame_height),
+            cameraMatrix=None,
+            distCoeffs=None
+        )
+
+        return float(reprojection_error_rms), camera_matrix, distortion_coefficients
+
+    def _caputre_calibration_sample(self):
+        detections = {}
+
+        for camera in self.cameraList: # Loop all Cameras
+            ok, frame = camera.capture.read() #Capture a frame
+
+            if not ok: # Check if captured frame is valid, if even one frame is invalide the whole capture failed
+                self.logger.log(f"Could not grab frame for calibration sample, from camera {camera.index}.")
+                return
+            
+            h, w = frame.shape[:2]  
+            self.logger.log(f"cam{camera.index} capture size = {w}x{h}")
+
+            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Turn the frame black and white
+            gray = 255 - gray # Invert the color of the frame, as the ChArCuo board I use is inverted (to save printer Ink)
+
+            h, w = gray.shape[:2]  
+            self.logger.log(f"cam{camera.index} capture size = {w}x{h}")
+
+            #This is apparently a fast check, bevor i detect the actuall board
+            corner, ids, _ = self.arcuo_detector.detectMarkers(gray) #Detect ChArCuo Markers
+            if ids is None or len(ids) == 0: #Check if Markers where found, if none where found the sample is invalid
+                self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.")
+                return
+            
+            # Detecting the Charcuo board and evaluating if its good as a sample
+            result = self.charuco_detector.detectBoard(gray)
+
+            charuco_corners = result[0]
+            charuco_ids = result[1]
+
+            if charuco_ids is None:
+                self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.")
+                return
+            
+            if len(charuco_ids) < MIN_CHARUCO_CORNERS:
+                self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.")
+                return
+            
+            # Adding the sample for the current camera to detections
+            detections[camera.index] = CharucoDetection(
+                camera_id = camera.index,
+                charuco_corners = charuco_corners,
+                charuco_ids=charuco_ids,
+                gray_frame = gray
+            )
+
+        # All cameras found valide markers so the sample capture was succesfull
+        self.calibration_samples.append(
+            CalibrationSample(detections=detections)
+        )
+
+        self.logger.log("Added calibration sample.")
+
+    def updateCameras(self):
+        self._clearLayout()
+
+        for camera_Object in self.cameraList:
+            camera_Setup_Widget = CameraSetupWidget(camera_Object)
+
+            self.scroll_Layout.addWidget(camera_Setup_Widget)
+            self.camera_Setup_Widget_List.append(camera_Setup_Widget)
+    
+    def _clearLayout(self):
+        self.camera_Setup_Widget_List.clear()
+        
+        while self.scroll_Layout.count():
+            item = self.scroll_Layout.takeAt(0)
+            widget = item.widget()
+            if widget is not None:
+                widget.setParent(None)
+                widget.deleteLater()
+

source/UI/SetupPageWidget.py

@@ -0,0 +1,228 @@
+import cv2, numpy as np
+import os
+
+
+from Tbd.helper import CameraObject
+from UI.UILogger import UILogger
+
+from PySide6.QtCore import Qt, QTimer
+from PySide6.QtGui import QImage, QPixmap
+from PySide6.QtWidgets import QLineEdit
+from PySide6.QtGui import QIntValidator
+from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QPushButton, QHBoxLayout
+from typing import List
+
+
+
+
+class SetupPage(QWidget):
+    """
+    Page that handles camera preview & cycling through cameras.
+    """
+
+    def __init__(self, cameraList: List[CameraObject], on_cameraList_changed, logger : UILogger):
+        super().__init__()
+
+        self.on_cameraList_changed = on_cameraList_changed
+        self.cameraList = cameraList
+        self.logger = logger
+        self.cameraObject = CameraObject(
+            capture=None,
+            index=None
+        )
+
+        # --- UI ---
+        self.setUpLayout = QVBoxLayout(self)
+
+        self.setUpLayout.addLayout(self._make_Setup_Controls())
+        self.setUpLayout.addLayout(self._make_Setup_Options())
+
+    def _make_Setup_Options(self):
+        setupOptions = QHBoxLayout()
+        setupOptions2 = QVBoxLayout()
+
+        self.cameraIndexLable = QLabel("Camera Index:")
+        self.cameraIndex = QLineEdit("0")
+        self.cameraIndex.setValidator(QIntValidator(0, 9999, self))
+
+        self.addCameraBTN = QPushButton("Add Camera")
+        self.nextCameraBTN = QPushButton("Next Camera")
+        self.previousCameraBTN = QPushButton("Previous Camera")
+
+        setupOptions.addWidget(self.cameraIndexLable)
+        setupOptions.addWidget(self.cameraIndex, 1)
+        setupOptions.addWidget(self.addCameraBTN)
+        setupOptions.addWidget(self.nextCameraBTN)
+        setupOptions.addWidget(self.previousCameraBTN)
+
+        self.nextCameraBTN.clicked.connect(self._nextIndex)
+        self.previousCameraBTN.clicked.connect(self._previousIndex)
+        #self.cameraIndex.editingFinished.connect(self._updateCamera)
+        self.addCameraBTN.clicked.connect(self._addCamera)
+
+        # ---
+        self.camera_preview = QLabel("No camera")
+        self.camera_preview.setAlignment(Qt.AlignCenter)
+        self.camera_preview.setMinimumSize(640, 360)
+        self.camera_preview.setStyleSheet("background: #222; color: #aaa;")
+
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self._updateCamera)
+
+        setupOptions2.addLayout(setupOptions)
+        setupOptions2.addWidget(self.camera_preview)
+
+
+        return setupOptions2
+    
+    def _addCamera(self):
+
+        if any(camera.index == int(self.cameraIndex.text()) for camera in self.cameraList):
+            print("Camera already added.")
+            return
+        
+        self.cameraList.append(self.cameraObject)
+        self._cameraList_changed()
+
+
+    def load_setup(self):
+        if (os.path.exists(self.setupFilePath.text())):
+            self.logger.log("Path found")
+        else:
+            self.logger.log("Path not found")
+
+            
+        indexToAdd =  [0, 3]
+        #self.cameraObject = CameraObject(
+        #    capture=None,
+        #    index=None
+        #)
+        for index in indexToAdd:
+            capture = cv2.VideoCapture(index, cv2.CAP_MSMF)
+            #capture = cv2.VideoCapture(index, cv2.CAP_DSHOW)
+
+            capture.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
+            capture.set(cv2.CAP_PROP_FPS, 30)
+            capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2592)
+            capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1944)
+
+            #capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2048.0)
+            #capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1536.0)
+            width = capture.get(cv2.CAP_PROP_FRAME_WIDTH)
+            height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT)
+            fps = capture.get(cv2.CAP_PROP_FPS)
+
+            fourcc = int(capture.get(cv2.CAP_PROP_FOURCC))
+            fourcc_str = "".join([chr((fourcc >> 8*i) & 0xFF) for i in range(4)])
+
+            print(f"Resolution: {width}x{height}")
+            print(f"FPS: {fps}")
+            print(f"Format (FOURCC): {fourcc_str}")
+            self.logger.log(f"Resolution: {width}x{height}")
+            self.logger.log(f"FPS: {fps}")
+            self.logger.log(f"Format (FOURCC): {fourcc_str}")
+            #currentcapture = cv2.VideoCapture(index, cv2.CAP_MSMF)
+            
+            #self.logger.log("Initial frame:", w, "x", h)
+            self.cameraList.append(CameraObject(
+                capture=capture,
+                index=index
+            ))
+            self._cameraList_changed()
+
+        
+        return
+
+        
+        
+    def save_setup(self):
+
+        print("ToDo")
+        return True
+    
+    def _cameraList_changed(self):
+        if self.on_cameraList_changed:
+            self.on_cameraList_changed()
+       
+    def _updateCamera(self):
+        if self.cameraObject.capture is None:
+            return
+        ret, frame = self.cameraObject.capture.read()
+        if not ret:
+            return
+
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        h, w, ch = frame_rgb.shape
+        bytes_per_line = ch * w
+        qimg = QImage(frame_rgb.data, w, h, bytes_per_line, QImage.Format_RGB888)
+        self.camera_preview.setPixmap(QPixmap.fromImage(qimg))
+        
+    def _nextIndex(self):
+        cameraIndex = int(self.cameraIndex.text())
+        self.cameraIndex.setText(str(cameraIndex + 1))
+
+        if self.open_camera():
+            if not self.timer.isActive():
+                self.timer.start(30)
+
+        self._updateCamera()
+    
+    def _previousIndex(self):
+        cameraIndex = int(self.cameraIndex.text())
+
+        if((cameraIndex - 1) < 0):
+            self.cameraIndex.setText("0")
+
+        self.cameraIndex.setText(str(cameraIndex - 1))
+
+        if self.open_camera():
+            if not self.timer.isActive():
+                self.timer.start(30)
+        
+        self._updateCamera()
+    
+    def open_camera(self):
+        index = int(self.cameraIndex.text())
+
+        self.release_camera()
+        self.cameraObject.capture = cv2.VideoCapture(index)
+        self.cameraObject.index = index
+
+        if not self.cameraObject.capture.isOpened():
+            self.cameraObject.capture = None
+            self.cameraObject.index = None
+            self.camera_preview.setText(f"Failed to open camera {index}")
+            return False
+        
+        self.camera_preview.setText("")
+
+        return True
+
+    def release_camera(self):
+        if self.cameraObject.capture is not None:
+            self.timer.stop()
+            self.cameraObject.capture.release()
+            self.cameraObject.capture = None
+            self.cameraObject.index = None
+            
+    def closeEvent(self, event):
+        self.release_camera()
+        event.accept()
+
+    def _make_Setup_Controls(self):
+        # Create Setup Input
+        setupFileLayout = QHBoxLayout()
+        self.setupFilePathLable = QLabel("Setupfile path:")
+        self.setupFilePath = QLineEdit()
+        self.loadBTN = QPushButton("Load")
+        self.saveBTN = QPushButton("Save")
+
+        setupFileLayout.addWidget(self.setupFilePathLable)
+        setupFileLayout.addWidget(self.setupFilePath, 1)
+        setupFileLayout.addWidget(self.loadBTN)
+        setupFileLayout.addWidget(self.saveBTN)
+
+        self.loadBTN.clicked.connect(self.load_setup)
+        self.saveBTN.clicked.connect(self.save_setup)
+
+        return setupFileLayout

source/UI/UILogger.py

@@ -0,0 +1,23 @@
+from PySide6.QtCore import QObject, Signal, Slot
+from PySide6.QtWidgets import QPlainTextEdit
+import winsound
+
+class UILogger(QObject):
+    append_line = Signal(str)
+
+    def __init__(self, widget: QPlainTextEdit):
+        super().__init__()
+
+        self.widget = widget
+        self.widget.setReadOnly(True)
+        self.widget.setMaximumBlockCount(2000)
+        self.append_line.connect(self._append)
+
+
+    @Slot(str)
+    def _append(self, text: str):
+        self.widget.appendPlainText(text)
+
+    def log(self, text:str):
+        self.append_line.emit(text)
+        winsound.PlaySound("SystemExclamation", winsound.SND_ALIAS | winsound.SND_ASYNC)

source/main.py

@@ -0,0 +1,164 @@
+import cv2, numpy as np
+import sys
+import os
+import json
+from pathlib import Path
+
+from Tbd.helper import default_profile, CameraObject
+from UI.SetupPageWidget import SetupPage
+from UI.HomePageWidget import HomeWindow
+from UI.GameWindow import GameWindow
+from UI.UILogger import UILogger
+from Tbd.TrackingController import TrackingController
+
+from PySide6.QtCore import Qt, QTimer
+from PySide6.QtGui import QImage, QPixmap
+from PySide6.QtWidgets import (
+    QApplication,
+    QMainWindow,
+    QStackedWidget,
+    QMessageBox,
+    QPlainTextEdit
+)
+from PySide6.QtWidgets import QLineEdit
+from PySide6.QtGui import QIntValidator
+from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QPushButton, QHBoxLayout, QSizePolicy
+from typing import List
+
+CONFIG_PATH = Path("config/profile_default.json")
+
+
+
+class MainWindow(QMainWindow):
+    """
+    Main control window: left sidebar + right content area.
+    """
+    cameraList = []
+
+    def __init__(self):
+        super().__init__()
+
+        self.P_mats = {}
+
+        self.setWindowTitle("Gaming Table Control") 
+        self.resize(1200, 1200)
+
+        # Central root widget
+        root = QWidget()
+        self.setCentralWidget(root)
+
+        self.main_layout = QVBoxLayout(root)
+        self.main_interaction_layout = QHBoxLayout()
+
+        # Create Logger
+        self.console = QPlainTextEdit()
+        self.logger = UILogger(self.console)
+
+        # ---------- Sidebar ----------
+        self.sidebar = QVBoxLayout()
+
+        self.btn_home = QPushButton("Home")
+        self.btn_start = QPushButton("Start Game")
+        self.btn_stop = QPushButton("Stop Game")
+        self.btn_setup = QPushButton("Setup Cameras")
+        self.btn_exit = QPushButton("Exit")
+
+        for btn in [self.btn_home, self.btn_start, self.btn_stop, self.btn_setup, self.btn_exit]:
+            btn.setMinimumHeight(40)
+            self.sidebar.addWidget(btn)
+
+        self.sidebar.addStretch(1)  # push buttons to top
+        
+        # ---------- Content area (stack) ----------
+        self.stack = QStackedWidget()
+
+        # Home page
+        home_page = QWidget()
+        self.homeWidget = HomeWindow(
+            cameraList=self.cameraList,
+            on_cameraList_changed=self.on_cameraList_changed,
+            P_mats=self.P_mats,
+            logger=self.logger
+        )
+
+        home_layout = QVBoxLayout(home_page)
+        home_label = QLabel("Welcome to the Gaming Table Control")
+        home_label.setAlignment(Qt.AlignCenter)
+
+        home_layout.addWidget(home_label)
+        home_layout.addWidget(self.homeWidget)
+
+        # Setup page
+        self.setup_page = SetupPage(
+            cameraList=self.cameraList,
+            on_cameraList_changed=self.on_cameraList_changed,
+            logger=self.logger
+        )
+
+        # Options page (placeholder)
+        stop_page = QWidget()
+        stop_layout = QVBoxLayout(stop_page)
+        stop_label = QLabel("The game was stopped")
+        stop_label.setAlignment(Qt.AlignCenter)
+        stop_layout.addWidget(stop_label)
+
+        # Add pages to stack
+        self.page_home_index = self.stack.addWidget(home_page)
+        self.page_setup_index = self.stack.addWidget(self.setup_page)
+        self.page_stop_index = self.stack.addWidget(stop_page)
+
+        # ---------- Assemble layout ----------
+        self.main_interaction_layout.addLayout(self.sidebar)      # left side
+        self.main_interaction_layout.addWidget(self.stack, 1)  # right side grows
+
+        self.main_layout.addLayout(self.main_interaction_layout)
+        self.main_layout.addWidget(self.console)
+
+        # ---------- Connections ----------
+        self.btn_home.clicked.connect(self.show_home)
+        self.btn_setup.clicked.connect(self.show_setup)
+        self.btn_stop.clicked.connect(self.show_stop)
+        self.btn_start.clicked.connect(self.start_game)
+        self.btn_exit.clicked.connect(self.close)
+
+        self.game_window = None
+
+    # --------------------
+    def on_cameraList_changed(self):
+        self.homeWidget.updateCameras()
+        self.logger.log("Camera List changed")
+        for camera in self.cameraList:
+            self.logger.log(str(camera.index))
+
+
+    def show_home(self):
+        self.stack.setCurrentIndex(self.page_home_index)
+
+    def show_setup(self):
+        self.stack.setCurrentIndex(self.page_setup_index)
+
+    def show_stop(self):
+        self.stack.setCurrentIndex(self.page_stop_index)
+
+        if(self.tracking) is not None:
+            self.tracking.stop()
+
+    def start_game(self):
+        self.tracking = TrackingController(
+            cameraList=self.cameraList,
+            P_mats=self.P_mats,
+            logger=self.logger,
+            tick_ms=30
+        )
+
+        self.tracking.start()
+
+def main():
+    app = QApplication(sys.argv)
+    win = MainWindow()
+    win.show()
+    sys.exit(app.exec())
+
+
+if __name__ == "__main__":
+    main()