Commit for Gitea

.vscode/launch.json

@@ -0,0 +1,12 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Run main.py",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/source/main.py",
+            "console": "integratedTerminal"
+        }
+    ]
+}

Stuff/Untitled-1.txt

@@ -1,15 +1,16 @@
-import cv2, numpy as np, time, fitz
+import cv2, numpy as np
-from calibration import object_calibration, draw_cal_circle
+from calibration import object_calibration
 from hand_detection import detect_hands, check_hand
 from helper import compute_distance
-from hud import find_hud_placement, draw_hud
+from source.UI.hud import find_hud_placement
 import math
 
-from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
+from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
+from config import FRAME_INTERVAL, TARGET_FPS
 from config import DEBUG
-
+from setup import InitCameras
-from hud import draw_hud_box, draw_pdf_page
+#from hud import draw_hud_box, draw_pdf_page
-
+import time
 
 # ----- Global State -----
 measuring = False             # measurement in progress
@@ -18,27 +19,22 @@ calibrating = False           # pinch-based calibration flag
 object_calibrating = False    # object calibration flag
 fingertip_idx_global = None   # last detected fingertip position
 
-
 hud_pos = None
 hud_rot = None
 register_hud = False
 
 # ----- Per-camera Processing -----
-def process_frame(frame):
+def process_frame(frame, camera_id: int = 0):
     global measuring, start_pt, calibrating
     global object_calibrating
     global fingertip_idx_global
     
     global register_hud, hud_pos, hud_rot
 
-    frame_out = cv2.flip(frame, 1)
+    frame_vis = cv2.flip(frame, 1)
-
-    frame_vis = frame_out.copy()
-    projection_debug = frame_out.copy()
 
     # Hand detection
-
+    results = detect_hands(frame_vis)
-    results = detect_hands(frame_out)
 
     fingertip_idx = None
     if results.multi_hand_landmarks:
@@ -52,83 +48,74 @@ def process_frame(frame):
         object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
         object_calibrating = False
     
-
+    # place hud
     res = None
     if hud_pos is None:
         res = find_hud_placement(frame_vis)
     else:
         res = hud_pos, hud_rot
 
-    if res is not None:
+    # create proj output
-        arrow_tip, dir_vec = res
+    projection_out = np.zeros_like(frame)
-        draw_hud(frame_vis, arrow_tip, dir_vec)
+    
+    # measuring
+    if measuring and start_pt and fingertip_idx_global:
+        cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
+        px = compute_distance(start_pt, fingertip_idx_global)
+        inch = px/PIXELS_PER_INCH; cm = inch*2.54
+        mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
+        cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
+
+    # hud
+    if res is not None or hud_pos:
+        if hud_pos:
+            arrow_tip = hud_pos
+            dir_vec = hud_rot
+        else:
+            arrow_tip, dir_vec = res
 
         dx, dy = dir_vec
         arrow_angle = math.atan2(dy, dx)   # result in radians
-        quad = draw_hud_box(frame_vis, arrow_tip, arrow_angle, size=(400,400))
+        #quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400))
-        draw_pdf_page(frame_vis, page_index=0, dst_quad=quad)
+        #draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
 
         if register_hud:
             hud_pos = arrow_tip
             hud_rot = dir_vec
             register_hud = False
 
-    return frame_vis
+    
+    mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0
+    frame_vis[mask] = projection_out[mask]
+
+    return frame_vis, projection_out
 
 # ----- Main ----- -------------------------------------------------------------------------------------
 def main():
-    global register_hud, hud_pos, hud_rot
+    all_captures = InitCameras()
-
+    
-    all_captures = []
+    print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)")
-    for idx in CAMERA_INDICES:
-        capture = cv2.VideoCapture(idx)
-        capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
-        capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
-        all_captures.append(capture)
-    if not all(capture.isOpened() for capture in all_captures):
-        print("Error: could not open all cameras")
-        return
-
     print("Press 'o' for circle calib, 'q' to quit.")
-    while True:
-        frames = [capture.read()[1] for capture in all_captures]
-        frame = next((f for f in frames if f is not None), None)
 
-        if frame is None:
+    last_time = time.time()
+    while True:
+        processed_frames = []
+
+        for camera_id, capture in enumerate(all_captures):
+            ok, frame = capture.read()
+            if not ok or frame is None:
+                continue
+
+            projection, debug = process_frame(frame, camera_id)
+            processed_frames.append((camera_id, projection, debug))
+
+        if not processed_frames:
             break
 
-        # process
+        for camera_id, projection, debug in processed_frames:
-        full_view = process_frame(frame)
+            if DEBUG:
-
+                cv2.imshow(f'Debug Output (cam {camera_id})', debug)
-        # create proj output
+            cv2.imshow(f'Projector Output (cam {camera_id})', projection)
-        projection_out = np.zeros_like(full_view)
-        
-        if measuring and start_pt and fingertip_idx_global:
-            cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
-            px = compute_distance(start_pt, fingertip_idx_global)
-            inch = px/PIXELS_PER_INCH; cm = inch*2.54
-            mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
-            cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-            cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
-
-        if hud_pos:
-            dx, dy = hud_rot
-            arrow_angle = math.atan2(dy, dx)   # result in radians
-            quad = draw_hud_box(projection_out, hud_pos, arrow_angle, size=(400,400))
-            draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
-            
-        # overlay proj onto debug
-        debug = full_view.copy()
-
-        # overlay measurement
-        if measuring and start_pt and fingertip_idx_global:
-            cv2.line(debug, start_pt, fingertip_idx_global, (255,255,0),2)
-            cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-            cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
-
-        # show windows
-        cv2.imshow('Debug Output', debug)
-        cv2.imshow('Projector Output', projection_out)
 
         key = cv2.waitKey(1) & 0xFF
 
@@ -139,13 +126,22 @@ def main():
             object_calibrating = True
             print("Entered circle calibration mode.")
         elif key == ord('j'):
+            global register_hud, hud_pos
             register_hud = True
             hud_pos = None
 
+        # ——— frame‐rate limiting ———
+        now     = time.time()
+        elapsed = now - last_time
+        to_wait = FRAME_INTERVAL - elapsed
+        if to_wait > 0:
+            time.sleep(to_wait)
+        last_time = time.time()
+
     for capture in all_captures:
         capture.release()
 
     cv2.destroyAllWindows()
 
 if __name__ == '__main__':
-    main()
+    main()

Table/.obsidian/app.json

@@ -0,0 +1 @@
+{}

Table/.obsidian/appearance.json

@@ -0,0 +1 @@
+{}

Table/.obsidian/core-plugins.json

@@ -0,0 +1,33 @@
+{
+  "file-explorer": true,
+  "global-search": true,
+  "switcher": true,
+  "graph": true,
+  "backlink": true,
+  "canvas": true,
+  "outgoing-link": true,
+  "tag-pane": true,
+  "footnotes": false,
+  "properties": false,
+  "page-preview": true,
+  "daily-notes": true,
+  "templates": true,
+  "note-composer": true,
+  "command-palette": true,
+  "slash-command": false,
+  "editor-status": true,
+  "bookmarks": true,
+  "markdown-importer": false,
+  "zk-prefixer": false,
+  "random-note": false,
+  "outline": true,
+  "word-count": true,
+  "slides": false,
+  "audio-recorder": false,
+  "workspaces": false,
+  "file-recovery": true,
+  "publish": false,
+  "sync": true,
+  "bases": true,
+  "webviewer": false
+}

Table/.obsidian/graph.json

@@ -0,0 +1,22 @@
+{
+  "collapse-filter": true,
+  "search": "",
+  "showTags": false,
+  "showAttachments": false,
+  "hideUnresolved": false,
+  "showOrphans": true,
+  "collapse-color-groups": true,
+  "colorGroups": [],
+  "collapse-display": true,
+  "showArrow": false,
+  "textFadeMultiplier": 0,
+  "nodeSizeMultiplier": 1,
+  "lineSizeMultiplier": 1,
+  "collapse-forces": true,
+  "centerStrength": 0.518713248970312,
+  "repelStrength": 10,
+  "linkStrength": 1,
+  "linkDistance": 250,
+  "scale": 1,
+  "close": true
+}

Table/.obsidian/workspace.json

@@ -0,0 +1,254 @@
+{
+  "main": {
+    "id": "833985cf0bef0982",
+    "type": "split",
+    "children": [
+      {
+        "id": "9d07647e9b05c8a5",
+        "type": "tabs",
+        "dimension": 50,
+        "children": [
+          {
+            "id": "777c49eacd333b57",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Table/Hand Tracking.md",
+                "mode": "source",
+                "source": false
+              },
+              "icon": "lucide-file",
+              "title": "Hand Tracking"
+            }
+          }
+        ]
+      },
+      {
+        "id": "ec54e744a9373b63",
+        "type": "tabs",
+        "dimension": 50,
+        "children": [
+          {
+            "id": "55f84d8c4a6e6b37",
+            "type": "leaf",
+            "state": {
+              "type": "graph",
+              "state": {},
+              "icon": "lucide-git-fork",
+              "title": "Graph view"
+            }
+          },
+          {
+            "id": "cdcef64cb6388441",
+            "type": "leaf",
+            "state": {
+              "type": "release-notes",
+              "state": {
+                "currentVersion": "1.10.6"
+              },
+              "icon": "lucide-book-up",
+              "title": "Release Notes 1.10.6"
+            }
+          },
+          {
+            "id": "48db892d028c7efb",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Table/Addons/USB Camera Case.md",
+                "mode": "source",
+                "source": false
+              },
+              "icon": "lucide-file",
+              "title": "USB Camera Case"
+            }
+          },
+          {
+            "id": "a871b6f7e2ef2b0b",
+            "type": "leaf",
+            "state": {
+              "type": "markdown",
+              "state": {
+                "file": "Table/General.md",
+                "mode": "source",
+                "source": false
+              },
+              "icon": "lucide-file",
+              "title": "General"
+            }
+          },
+          {
+            "id": "ec002b0aa870cc5d",
+            "type": "leaf",
+            "state": {
+              "type": "release-notes",
+              "state": {
+                "currentVersion": "1.11.7"
+              },
+              "icon": "lucide-book-up",
+              "title": "Release Notes 1.11.7"
+            }
+          }
+        ],
+        "currentTab": 4
+      }
+    ],
+    "direction": "vertical"
+  },
+  "left": {
+    "id": "f6a806b37d1ae6eb",
+    "type": "split",
+    "children": [
+      {
+        "id": "9148cc17fa29c8df",
+        "type": "tabs",
+        "children": [
+          {
+            "id": "1c44bf34acded4f0",
+            "type": "leaf",
+            "state": {
+              "type": "file-explorer",
+              "state": {
+                "sortOrder": "alphabetical",
+                "autoReveal": false
+              },
+              "icon": "lucide-folder-closed",
+              "title": "Files"
+            }
+          },
+          {
+            "id": "85e41cb90d92a857",
+            "type": "leaf",
+            "state": {
+              "type": "search",
+              "state": {
+                "query": "",
+                "matchingCase": false,
+                "explainSearch": false,
+                "collapseAll": false,
+                "extraContext": false,
+                "sortOrder": "alphabetical"
+              },
+              "icon": "lucide-search",
+              "title": "Search"
+            }
+          },
+          {
+            "id": "8ada34bb1b02dff6",
+            "type": "leaf",
+            "state": {
+              "type": "bookmarks",
+              "state": {},
+              "icon": "lucide-bookmark",
+              "title": "Bookmarks"
+            }
+          }
+        ]
+      }
+    ],
+    "direction": "horizontal",
+    "width": 300
+  },
+  "right": {
+    "id": "b343b576de17e5fa",
+    "type": "split",
+    "children": [
+      {
+        "id": "a960a5dad3743e2e",
+        "type": "tabs",
+        "children": [
+          {
+            "id": "d06b72b904a43a4e",
+            "type": "leaf",
+            "state": {
+              "type": "backlink",
+              "state": {
+                "collapseAll": false,
+                "extraContext": false,
+                "sortOrder": "alphabetical",
+                "showSearch": false,
+                "searchQuery": "",
+                "backlinkCollapsed": false,
+                "unlinkedCollapsed": true
+              },
+              "icon": "links-coming-in",
+              "title": "Backlinks"
+            }
+          },
+          {
+            "id": "906e21e00a9b4d73",
+            "type": "leaf",
+            "state": {
+              "type": "outgoing-link",
+              "state": {
+                "linksCollapsed": false,
+                "unlinkedCollapsed": true
+              },
+              "icon": "links-going-out",
+              "title": "Outgoing links"
+            }
+          },
+          {
+            "id": "6c98ae458072df7c",
+            "type": "leaf",
+            "state": {
+              "type": "tag",
+              "state": {
+                "sortOrder": "frequency",
+                "useHierarchy": true,
+                "showSearch": false,
+                "searchQuery": ""
+              },
+              "icon": "lucide-tags",
+              "title": "Tags"
+            }
+          },
+          {
+            "id": "0ce33e15635006fc",
+            "type": "leaf",
+            "state": {
+              "type": "outline",
+              "state": {
+                "followCursor": false,
+                "showSearch": false,
+                "searchQuery": ""
+              },
+              "icon": "lucide-list",
+              "title": "Outline"
+            }
+          }
+        ]
+      }
+    ],
+    "direction": "horizontal",
+    "width": 300,
+    "collapsed": true
+  },
+  "left-ribbon": {
+    "hiddenItems": {
+      "switcher:Open quick switcher": false,
+      "graph:Open graph view": false,
+      "canvas:Create new canvas": false,
+      "daily-notes:Open today's daily note": false,
+      "templates:Insert template": false,
+      "command-palette:Open command palette": false,
+      "bases:Create new base": false
+    }
+  },
+  "active": "ec002b0aa870cc5d",
+  "lastOpenFiles": [
+    "Table/Hand Tracking.md",
+    "Table/Addons/Addons.md",
+    "Table/Addons/Cable Holder.md",
+    "Table/Addons/Switch.md",
+    "Table/Addons/USB Camera Case.md",
+    "Table/Addons/Universal Mount.md",
+    "Untitled",
+    "Table/General.md",
+    "Table/Addons",
+    "Table/Addons",
+    "Table"
+  ]
+}

Table/Table/Addons/Addons.md

@@ -0,0 +1,6 @@
+
+| Addon            | Description             |
+| ---------------- | ----------------------- |
+| [[Switch]]       | A casing for the switch |
+| [[Cable Holder]] |                         |
+

Table/Table/Addons/Switch.md

@@ -0,0 +1,9 @@
+
+| Depth | 11   |
+| ----- | ---- |
+| Widht | 9.45 |
+| Hight | 2.9  |
+|       |      |
+|       |      |
+|       |      |
+|       |      |

Table/Table/General.md

@@ -0,0 +1,7 @@
+Gaming Table for Warhammer and Stuff.
+
+[[Addons]]
+
+
+
+[[Hand Tracking]]

Test/HUDs/DarkAngles/DarkAngles.json

@@ -0,0 +1,9 @@
+{
+  "type": "PDFContainer",
+  "sources": [
+    {
+      "contenttype": "PDF",
+      "source": "warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf"
+    }
+  ]
+}

Test/HUDs/DarkAngles/FuckThisShit.txt

@@ -0,0 +1 @@
+FuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txt

Test/test.json

@@ -0,0 +1,163 @@
+{
+  "version": 1,
+  "cameras": [
+    {
+      "index": 0,
+      "pxWidth": 2048,
+      "pxHeight": 1536,
+      "camera_matrix": [
+        [
+          1637.301826012801,
+          0.0,
+          1115.6922895709363
+        ],
+        [
+          0.0,
+          1637.3016923198538,
+          660.417851928144
+        ],
+        [
+          0.0,
+          0.0,
+          1.0
+        ]
+      ],
+      "distortion_coefficients": [
+        [
+          0.1880726328811075,
+          -0.3711284739403174,
+          -0.019390902558209716,
+          0.009835323140375036,
+          0.42533165931899103
+        ]
+      ],
+      "rotation_matrix_world_to_camera": [
+        [
+          1.0,
+          0.0,
+          0.0
+        ],
+        [
+          0.0,
+          1.0,
+          0.0
+        ],
+        [
+          0.0,
+          0.0,
+          1.0
+        ]
+      ],
+      "translation_vector_world_to_camera": [
+        [
+          0.0
+        ],
+        [
+          0.0
+        ],
+        [
+          0.0
+        ]
+      ],
+      "camera_projection_matrix": [
+        [
+          1637.301826012801,
+          0.0,
+          1115.6922895709363,
+          0.0
+        ],
+        [
+          0.0,
+          1637.3016923198538,
+          660.417851928144,
+          0.0
+        ],
+        [
+          0.0,
+          0.0,
+          1.0,
+          0.0
+        ]
+      ]
+    },
+    {
+      "index": 3,
+      "pxWidth": 2048,
+      "pxHeight": 1536,
+      "camera_matrix": [
+        [
+          1629.3953951276987,
+          0.0,
+          946.5260213900043
+        ],
+        [
+          0.0,
+          1608.3790414523598,
+          656.5060407239171
+        ],
+        [
+          0.0,
+          0.0,
+          1.0
+        ]
+      ],
+      "distortion_coefficients": [
+        [
+          0.19251720938106318,
+          -0.21522629593789278,
+          -0.02730155124857322,
+          -0.03890715094594556,
+          0.21680710398239586
+        ]
+      ],
+      "rotation_matrix_world_to_camera": [
+        [
+          0.08276206632981453,
+          -0.8544623995610863,
+          0.5128785900319194
+        ],
+        [
+          0.720047607579738,
+          0.40706667659536877,
+          0.5619859105211598
+        ],
+        [
+          -0.6889716127646442,
+          0.3227858865283794,
+          0.6489432858598454
+        ]
+      ],
+      "translation_vector_world_to_camera": [
+        [
+          -176.11272583763233
+        ],
+        [
+          -564.9970879948509
+        ],
+        [
+          471.9913794371197
+        ]
+      ],
+      "camera_projection_matrix": [
+        [
+          -517.2774297117204,
+          -1086.7318582180455,
+          1449.923719330272,
+          159794.8580057718
+        ],
+        [
+          705.7954552118755,
+          866.628395475963,
+          1329.9215473279708,
+          -598864.2830424493
+        ],
+        [
+          -0.6889716127646442,
+          0.3227858865283794,
+          0.6489432858598454,
+          471.9913794371197
+        ]
+      ]
+    }
+  ]
+}

Test/udp_test.py

@@ -0,0 +1,8 @@
+import socket, struct, time
+
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.sendto(
+    struct.pack("<4sBIQB", b"HAND", 1, 123, int(time.time()*1000), 0),
+    ("127.0.0.1", 9000)
+)
+print("sent")

helper.py

@@ -1,21 +0,0 @@
-import numpy as np
-import cv2
-
-# ----- Helper Functions -----
-def compute_distance(p1, p2):
-    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
-
-# 4) compute straightness per finger
-def finger_straight(i_tip, i_pip, i_mcp):
-    d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip)))
-    d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp)))
-    return float(np.clip(d1/(d2+1e-3), 0, 1))
-
-def yes_no():
-    while True:
-        key = cv2.waitKey(1) & 0xFF
-
-        if key == ord("y"):
-            return True
-        if key == ord("n"):
-            return False

hud.py

@@ -1,61 +0,0 @@
-import cv2, numpy as np
-from config import LOWER_SHAPE, UPPER_SHAPE
-import cv2
-import numpy as np
-import time
-import fitz  # PyMuPDF for PDF rendering
-
-
-
-def find_hud_placement(frame_vis):
-    # ----- Arrow Shape Detection & HUD Placement -----
-    # 1) Build a clean orange mask
-    hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
-    mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE)
-    kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
-    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern)
-    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern)
-
-    # 2) Find and filter contours
-    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-    for cnt in contours:
-        area = cv2.contourArea(cnt)
-        if area < 1000:
-            continue
-
-        # Approximate to polygon and require exactly 5 corners
-        peri = cv2.arcLength(cnt, True)
-        approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)
-        if len(approx) != 5:
-            continue
-
-        pts = approx.reshape(-1,2)
-        centroid = np.mean(pts, axis=0)
-
-        # Find the arrow tip as the corner farthest from centroid
-        dists = [np.linalg.norm(pt - centroid) for pt in pts]
-        tip_pt = pts[int(np.argmax(dists))]
-        arrow_tip = (int(tip_pt[0]), int(tip_pt[1]))
-
-        # Place the HUD box in the arrow’s pointing direction
-        dir_vec = tip_pt - centroid
-
-        #ToDo remove
-        cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2)
-
-        return arrow_tip, dir_vec
-
-# hud.py
-class HUD:
-    def __init__(self, marker_id, pdf_pages, default_page=0):
-        self.id         = marker_id
-        self.page_index = default_page
-        self.pdf_pages  = pdf_pages    # list of preloaded page images
-        self.ref_quad   = None         # the 4-corner quad where to draw the PDF
-        self.last_seen  = 0            # for timeout if arrow goes away
-
-    def update(self, marker):
-        return
-
-    def draw(self, frame):
-        return

main.py

@@ -1,147 +0,0 @@
-import cv2, numpy as np
-from calibration import object_calibration
-from hand_detection import detect_hands, check_hand
-from helper import compute_distance
-from hud import find_hud_placement
-import math
-
-from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
-from config import FRAME_INTERVAL, TARGET_FPS
-from config import DEBUG
-from setup import InitCameras
-#from hud import draw_hud_box, draw_pdf_page
-import time
-
-# ----- Global State -----
-measuring = False             # measurement in progress
-start_pt = None               # measurement start point
-calibrating = False           # pinch-based calibration flag
-object_calibrating = False    # object calibration flag
-fingertip_idx_global = None   # last detected fingertip position
-
-hud_pos = None
-hud_rot = None
-register_hud = False
-
-# ----- Per-camera Processing -----
-def process_frame(frame, camera_id: int = 0):
-    global measuring, start_pt, calibrating
-    global object_calibrating
-    global fingertip_idx_global
-    
-    global register_hud, hud_pos, hud_rot
-
-    frame_vis = cv2.flip(frame, 1)
-
-    # Hand detection
-    results = detect_hands(frame_vis)
-
-    fingertip_idx = None
-    if results.multi_hand_landmarks:
-        hand = results.multi_hand_landmarks[0]
-        fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating)
-
-    fingertip_idx_global = fingertip_idx
- 
-    # object calibration
-    if object_calibrating and fingertip_idx is not None:
-        object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
-        object_calibrating = False
-    
-    # place hud
-    res = None
-    if hud_pos is None:
-        res = find_hud_placement(frame_vis)
-    else:
-        res = hud_pos, hud_rot
-
-    # create proj output
-    projection_out = np.zeros_like(frame)
-    
-    # measuring
-    if measuring and start_pt and fingertip_idx_global:
-        cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
-        px = compute_distance(start_pt, fingertip_idx_global)
-        inch = px/PIXELS_PER_INCH; cm = inch*2.54
-        mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
-        cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
-
-    # hud
-    if res is not None or hud_pos:
-        if hud_pos:
-            arrow_tip = hud_pos
-            dir_vec = hud_rot
-        else:
-            arrow_tip, dir_vec = res
-
-        dx, dy = dir_vec
-        arrow_angle = math.atan2(dy, dx)   # result in radians
-        #quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400))
-        #draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
-
-        if register_hud:
-            hud_pos = arrow_tip
-            hud_rot = dir_vec
-            register_hud = False
-
-    
-    mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0
-    frame_vis[mask] = projection_out[mask]
-
-    return frame_vis, projection_out
-
-# ----- Main ----- -------------------------------------------------------------------------------------
-def main():
-    all_captures = InitCameras()
-    
-    print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)")
-    print("Press 'o' for circle calib, 'q' to quit.")
-
-    last_time = time.time()
-    while True:
-        processed_frames = []
-
-        for camera_id, capture in enumerate(all_captures):
-            ok, frame = capture.read()
-            if not ok or frame is None:
-                continue
-
-            projection, debug = process_frame(frame, camera_id)
-            processed_frames.append((camera_id, projection, debug))
-
-        if not processed_frames:
-            break
-
-        for camera_id, projection, debug in processed_frames:
-            if DEBUG:
-                cv2.imshow(f'Debug Output (cam {camera_id})', debug)
-            cv2.imshow(f'Projector Output (cam {camera_id})', projection)
-
-        key = cv2.waitKey(1) & 0xFF
-
-        if key == ord('q'):
-            break
-        elif key == ord('o'):
-            global object_calibrating
-            object_calibrating = True
-            print("Entered circle calibration mode.")
-        elif key == ord('j'):
-            global register_hud, hud_pos
-            register_hud = True
-            hud_pos = None
-
-        # ——— frame‐rate limiting ———
-        now     = time.time()
-        elapsed = now - last_time
-        to_wait = FRAME_INTERVAL - elapsed
-        if to_wait > 0:
-            time.sleep(to_wait)
-        last_time = time.time()
-
-    for capture in all_captures:
-        capture.release()
-
-    cv2.destroyAllWindows()
-
-if __name__ == '__main__':
-    main()

setup.py

@@ -1,81 +0,0 @@
-import cv2
-import numpy as np
-
-from config import CAMERA_INDICES_TO_CHECK, FRAME_WIDTH, FRAME_HEIGHT
-
-def _preview_camera(capture: cv2.VideoCapture, cameraID: int) -> str:
-    """Show a live preview and return the user's choice: 'add', 'skip', or 'quit'."""
-    window_name = f"Camera {cameraID}"
-    cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
-    print(f"[Camera {cameraID}] Press 'y' to add, 'n' to skip, 'q' to stop scanning.")
-
-    try:
-        while True:
-            ok, frame = capture.read()
-            if not ok:
-                print(f"[Camera {cameraID}] Failed to read frame; skipping.")
-                return "skip"
-
-            cv2.imshow(window_name, frame)
-
-            key = cv2.waitKey(1) & 0xFF
-
-            if key == ord("y"):
-                print(f"[Camera {cameraID}] Selected.")
-                return "add"
-            if key == ord("n"):
-                print(f"[Camera {cameraID}] Skipped.")
-                return "skip"
-            if key == ord("q") or key == 27:  # 27 == ESC
-                print(f"[Camera {cameraID}] Stopping camera scan.")
-                return "quit"
-    finally:
-        cv2.destroyWindow(window_name)
-
-
-def InitCameras():
-    allCaptures = []
-
-    setup_window = "Camera Setup"
-    instructions = np.zeros((240, 560, 3), dtype=np.uint8)
-    cv2.putText(instructions, "Camera Setup", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 200, 255), 2)
-    cv2.putText(instructions, "Each camera will preview in its own window.", (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
-    cv2.putText(instructions, "Use 'y' to add, 'n' to skip, 'q'/ESC to finish.", (20, 125), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
-    cv2.putText(instructions, "Close this window when you're done.", (20, 160), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
-
-    cv2.namedWindow(setup_window, cv2.WINDOW_AUTOSIZE)
-    cv2.imshow(setup_window, instructions)
-    cv2.waitKey(1)
-
-    try:
-        for cameraID in range(CAMERA_INDICES_TO_CHECK):
-            capture = cv2.VideoCapture(cameraID)
-
-            if not capture.isOpened():
-                capture.release()
-                continue
-
-            capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
-            capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
-
-            decision = _preview_camera(capture, cameraID)
-
-            if decision == "add":
-                allCaptures.append(capture)
-            else:
-                capture.release()
-
-            if decision == "quit":
-                break
-
-        if not all(capture.isOpened() for capture in allCaptures):
-            print("Error: could not open all selected cameras.")
-            return
-
-        if not allCaptures:
-            print("No cameras selected.")
-            return
-
-        return allCaptures
-    finally:
-        cv2.destroyWindow(setup_window)

source/Helpers/CHArUco_version_test.py

@@ -0,0 +1,5 @@
+import cv2
+print("OpenCV:", cv2.__version__)
+print("has aruco:", hasattr(cv2, "aruco"))
+print("has CharucoBoard:", hasattr(cv2.aruco, "CharucoBoard"))
+print("has ArucoDetector:", hasattr(cv2.aruco, "ArucoDetector"))

source/Helpers/ChArUco.py

@@ -0,0 +1,32 @@
+import cv2
+import numpy as np
+
+# ---- Board parameters (CHANGE THESE ONLY IF YOU REPRINT) ----
+squares_x = 6        # number of chessboard squares in X
+squares_y = 9        # number of chessboard squares in Y
+square_length = 25  # mm
+marker_length = 18  # mm (must be < square_length)
+dictionary_id = cv2.aruco.DICT_4X4_50
+dpi = 300            # print DPI
+# ------------------------------------------------------------
+
+dictionary = cv2.aruco.getPredefinedDictionary(dictionary_id)
+board = cv2.aruco.CharucoBoard(
+    (squares_x, squares_y),
+    square_length,
+    marker_length,
+    dictionary
+)
+
+# Convert physical size (mm) to pixels for printing
+mm_to_inch = 1 / 25.4
+width_mm = squares_x * square_length
+height_mm = squares_y * square_length
+
+width_px = int(width_mm * mm_to_inch * dpi)
+height_px = int(height_mm * mm_to_inch * dpi)
+
+img = board.generateImage((width_px, height_px))
+img = 255 - img
+cv2.imwrite("H:\Table\charuco_A4.png", img)
+print("Saved charuco_A4.png")

source/Helpers/ChArUco_confidence_Test.py

@@ -0,0 +1,220 @@
+"""
+ChArUco detection confidence test (OpenCV)
+
+What it does:
+- Opens one or multiple cameras
+- Detects ArUco markers and interpolated ChArUco corners
+- Draws overlays
+- Prints a simple "confidence" score per camera:
+    markers_found, charuco_corners_found, and a normalized confidence value
+
+Requirements:
+- OpenCV built with aruco module (opencv-contrib-python)
+    pip install opencv-contrib-python
+"""
+
+import cv2
+import numpy as np
+import time
+
+# ----------------------------
+# USER SETTINGS (match your printed board!)
+# ----------------------------
+CAM_IDS = [0, 2]                 # set to [0] for single camera test, or [0,2,3,...]
+USE_DSHOW_ON_WINDOWS = True      # good for Windows
+RESOLUTION = (3264, 2448)        # (width, height) if your cameras support it; else set None
+DICT_ID = cv2.aruco.DICT_4X4_50
+
+SQUARES_X = 6                    # number of chessboard squares in X
+SQUARES_Y = 9                    # number of chessboard squares in Y
+SQUARE_LEN_MM = 25.0             # square size in mm
+MARKER_LEN_MM = 18.0             # marker size in mm (must be < square)
+
+# Confidence thresholds (tune if needed)
+MIN_MARKERS_OK = 4               # markers to consider "good"
+MIN_CHARUCO_OK = 15              # charuco corners to consider "good"
+
+# Display
+WINDOW_SCALE = 0.5               # downscale for display if res is huge (0.5 = half size)
+PRINT_EVERY_SEC = 0.5
+# ----------------------------
+
+
+def open_camera(cam_id: int):
+    backend = cv2.CAP_DSHOW if (USE_DSHOW_ON_WINDOWS and hasattr(cv2, "CAP_DSHOW")) else 0
+    cap = cv2.VideoCapture(cam_id, backend)
+    if not cap.isOpened():
+        return None
+
+    if RESOLUTION is not None:
+        w, h = RESOLUTION
+        cap.set(cv2.CAP_PROP_FRAME_WIDTH, float(w))
+        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, float(h))
+
+    return cap
+
+
+def compute_confidence(markers: int, charuco: int) -> float:
+    """
+    Simple normalized confidence heuristic:
+    - markers contribute up to MIN_MARKERS_OK
+    - charuco corners contribute up to MIN_CHARUCO_OK
+    """
+    m = min(markers / max(MIN_MARKERS_OK, 1), 1.0)
+    c = min(charuco / max(MIN_CHARUCO_OK, 1), 1.0)
+    # Weighted: charuco corners matter more for calibration quality
+    return 0.35 * m + 0.65 * c
+
+
+def main():
+    dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID)
+    board = cv2.aruco.CharucoBoard(
+        (SQUARES_X, SQUARES_Y),
+        SQUARE_LEN_MM,
+        MARKER_LEN_MM,
+        dictionary
+    )
+
+    # Detector parameters
+    detector_params = cv2.aruco.DetectorParameters()
+    # You can tweak these if detection is unstable:
+    # detector_params.adaptiveThreshWinSizeMin = 3
+    # detector_params.adaptiveThreshWinSizeMax = 23
+    # detector_params.adaptiveThreshWinSizeStep = 10
+
+    detector = cv2.aruco.ArucoDetector(dictionary, detector_params)
+
+    caps = {}
+    for cam_id in CAM_IDS:
+        cap = open_camera(cam_id)
+        if cap is None:
+            print(f"[ERR] Could not open camera {cam_id}")
+        else:
+            caps[cam_id] = cap
+            print(f"[OK] Opened camera {cam_id}")
+
+    if not caps:
+        print("No cameras opened. Exiting.")
+        return
+
+    last_print = 0.0
+
+    print("\nControls:")
+    print("  ESC = quit")
+    print("  Space = print one-shot stats immediately\n")
+
+    try:
+        while True:
+            frames_vis = []
+            stats = {}
+
+            for cam_id, cap in caps.items():
+                ok, frame = cap.read()
+                if not ok or frame is None:
+                    stats[cam_id] = (0, 0, 0.0)
+                    continue
+
+                gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+                gray = 255 - gray
+                # Detect markers
+                corners, ids, rejected = detector.detectMarkers(gray)
+                markers_found = 0 if ids is None else len(ids)
+
+                # Draw markers
+                vis = frame.copy()
+                if ids is not None:
+                    cv2.aruco.drawDetectedMarkers(vis, corners, ids)
+
+                # Interpolate ChArUco corners (requires some markers found)
+                charuco_found = 0
+                if ids is not None and len(ids) > 0:
+                    charuco_detector = cv2.aruco.CharucoDetector(board)
+
+                    charuco_corners = None
+                    charuco_ids = None
+
+                    if ids is not None and len(ids) > 0:
+                        res = charuco_detector.detectBoard(gray)
+
+                        # OpenCV versions differ in what they return; handle both safely.
+                        # Common patterns:
+                        #   (charucoCorners, charucoIds, markerCorners, markerIds)
+                        #   (charucoCorners, charucoIds, rejectedMarkerCandidates)
+                        #   (charucoCorners, charucoIds, ...)
+                        charuco_corners = res[0] if len(res) > 0 else None
+                        charuco_ids = res[1] if len(res) > 1 else None
+                    charuco_found = 0
+                    if charuco_ids is not None:
+                        charuco_found = len(charuco_ids)
+                        cv2.aruco.drawDetectedCornersCharuco(vis, charuco_corners, charuco_ids)
+
+                conf = compute_confidence(markers_found, charuco_found)
+                stats[cam_id] = (markers_found, charuco_found, conf)
+
+                # Overlay text
+                h, w = vis.shape[:2]
+                lines = [
+                    f"Cam {cam_id}",
+                    f"Markers: {markers_found}",
+                    f"ChArUco corners: {charuco_found}",
+                    f"Confidence: {conf:.2f}",
+                    "GOOD" if (markers_found >= MIN_MARKERS_OK and charuco_found >= MIN_CHARUCO_OK) else "MOVE / LIGHT / FOCUS"
+                ]
+
+                y = 30
+                for line in lines:
+                    cv2.putText(vis, line, (20, y), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0) if "GOOD" in line else (0, 200, 255), 2)
+                    y += 32
+
+                # Downscale for display
+                if WINDOW_SCALE != 1.0:
+                    vis = cv2.resize(vis, None, fx=WINDOW_SCALE, fy=WINDOW_SCALE, interpolation=cv2.INTER_AREA)
+
+                frames_vis.append(vis)
+
+            # Combine displays
+            if frames_vis:
+                # Stack horizontally; if many cams, wrap to multiple rows
+                max_per_row = 3
+                rows = []
+                for i in range(0, len(frames_vis), max_per_row):
+                    row = frames_vis[i:i + max_per_row]
+                    # pad heights
+                    max_h = max(img.shape[0] for img in row)
+                    padded = []
+                    for img in row:
+                        if img.shape[0] < max_h:
+                            pad = max_h - img.shape[0]
+                            img = cv2.copyMakeBorder(img, 0, pad, 0, 0, cv2.BORDER_CONSTANT, value=(0, 0, 0))
+                        padded.append(img)
+                    rows.append(np.hstack(padded))
+                grid = np.vstack(rows)
+
+                cv2.imshow("ChArUco Confidence Test", grid)
+
+            key = cv2.waitKey(1) & 0xFF
+            now = time.time()
+
+            if key == 27:  # ESC
+                break
+
+            if key == 32:  # Space
+                last_print = 0  # force print now
+
+            if now - last_print >= PRINT_EVERY_SEC:
+                last_print = now
+                # Print compact stats
+                msg = " | ".join(
+                    f"cam{cid}: M={m} C={c} conf={conf:.2f}"
+                    for cid, (m, c, conf) in sorted(stats.items())
+                )
+                print(msg)
+
+    finally:
+        for cap in caps.values():
+            cap.release()
+        cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()

source/Helpers/HandUdpSender.py

@@ -0,0 +1,57 @@
+import socket
+import struct
+import time
+import numpy as np
+
+class HandUdpSender:
+    MAGIC = b'HAND'
+    VERSION = 2
+
+    def __init__(self, host="127.0.0.1", port=9000):
+        self.addr = (host, port)
+        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.seq = 0
+
+    def _encode_handedness(self, s: str) -> int:
+        if s == "Left":
+            return 1
+        if s == "Right":
+            return 2
+        return 0
+
+    def send_hands3d(self, hands3d: list[dict]):
+        """
+        hands3d: list of dicts:
+        {
+            "landmarks": (21,3) np.ndarray float,
+            "handedness": "Left"/"Right"/"Unknown",
+            "confidence": float (0..1)
+        }
+        Sends one UDP datagram containing all hands.
+        """
+        self.seq += 1
+        ts_ms = int(time.time() * 1000)
+
+        hand_count = min(len(hands3d), 255)
+
+        # Header: magic(4s), version(B), seq(I), ts(Q), hand_count(B)
+        packet = bytearray()
+        packet += struct.pack("<4sBIQB", self.MAGIC, self.VERSION, self.seq, ts_ms, hand_count)
+
+        for hid in range(hand_count):
+            h = hands3d[hid]
+            pts = np.asarray(h["landmarks"], dtype=np.float32)
+            if pts.shape != (21, 3):
+                continue
+
+            point_count = 21
+            handedness_code = self._encode_handedness(str(h.get("handedness", "Unknown")))
+            confidence = float(h.get("confidence", 0.0))
+
+            # per-hand header: hand_id(B), point_count(B), handedness(B), confidence(f)
+            packet += struct.pack("<BBBf", hid, point_count, handedness_code, confidence)
+
+            # points: 21*3 float32
+            packet += pts.tobytes(order="C")
+
+        self.sock.sendto(packet, self.addr)

source/Helpers/saveManager.py

@@ -0,0 +1,77 @@
+import json
+from pathlib import Path
+import numpy as np
+from Tbd.helper import  CameraObject
+
+def _np_to_list(a: np.ndarray):
+    return None if a is None else a.tolist()
+
+def save_calibration_json(camera_list, file_path: str, logger=None):
+    payload = {
+        "version": 1,
+        "cameras": []
+    }
+
+    for cam in camera_list:
+        payload["cameras"].append({
+            "index": int(cam.index),
+            "pxWidth": int(cam.pxWidth),
+            "pxHeight": int(cam.pxHeight),
+
+            "camera_matrix": _np_to_list(cam.camera_matrix),
+            "distortion_coefficients": _np_to_list(cam.distortion_coefficients),
+
+            "rotation_matrix_world_to_camera": _np_to_list(cam.rotation_matrix_world_to_camera),
+            "translation_vector_world_to_camera": _np_to_list(cam.translation_vector_world_to_camera),
+
+            "camera_projection_matrix": _np_to_list(cam.camera_projection_matrix),
+        })
+
+    Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(file_path, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2)
+
+    if logger:
+        logger.log(f"[Calibration] Saved calibration to {file_path}")
+
+def _list_to_np(x, shape=None):
+    if x is None:
+        return None
+    a = np.array(x, dtype=np.float64)
+    if shape is not None:
+        a = a.reshape(shape)
+    return a
+
+def load_calibration_json(file_path: str, logger=None):
+    file_path = str(file_path)
+    if not Path(file_path).exists():
+        raise FileNotFoundError(file_path)
+
+    with open(file_path, "r", encoding="utf-8") as f:
+        payload = json.load(f)
+
+    cams = []
+    for c in payload.get("cameras", []):
+        cam = CameraObject(
+            capture=None,
+            index=int(c["index"]),
+            pxWidth=int(c.get("pxWidth", 1536)),
+            pxHeight=int(c.get("pxHeight", 2048)),
+        )
+
+        cam.camera_matrix = _list_to_np(c.get("camera_matrix"), shape=(3,3))
+        cam.distortion_coefficients = _list_to_np(c.get("distortion_coefficients"))  # keep native shape
+
+        cam.rotation_matrix_world_to_camera = _list_to_np(c.get("rotation_matrix_world_to_camera"), shape=(3,3))
+        tv = _list_to_np(c.get("translation_vector_world_to_camera"))
+        if tv is not None:
+            cam.translation_vector_world_to_camera = tv.reshape(3,1)
+
+        cam.camera_projection_matrix = _list_to_np(c.get("camera_projection_matrix"), shape=(3,4))
+        cam.load_video_capture()
+        cams.append(cam)
+
+    if logger:
+        logger.log(f"[Calibration] Loaded calibration from {file_path} ({len(cams)} cameras)")
+
+    return cams

source/Tbd/TrackingController.py

@@ -0,0 +1,260 @@
+import numpy as np
+import cv2
+from PySide6.QtCore import QObject, QTimer
+import mediapipe as mp
+from Tbd.helper import Hand2D
+from collections import defaultdict
+from Helpers.HandUdpSender import HandUdpSender
+
+import struct
+import time
+
+class TrackingController(QObject):
+    def __init__(self, cameraList, P_mats, logger=None, tick_ms=30):
+        super().__init__()
+
+        self.cameraList = cameraList
+        self.P_mats = P_mats
+        self.logger = logger
+
+        self.udp = HandUdpSender(host="127.0.0.1", port=9000)
+
+        self.timer = QTimer(self)
+        self.timer.timeout.connect(self._tick)
+        self.tick_ms = tick_ms
+
+        self.running = False
+
+        # init mediapipe here (or inject it)
+
+        self.mp_hands = mp.solutions.hands
+        self.hands = self.mp_hands.Hands(
+            static_image_mode=False,
+            max_num_hands=4,
+            model_complexity=1,
+            min_detection_confidence=0.5,
+            min_tracking_confidence=0.5,
+        )
+
+    def start(self):
+        # Guard: need projections
+        self.logger.log("Amount of P_mats:" + str(len(self.P_mats)))
+        if not self.P_mats or len(self.P_mats) < 2:
+            if self.logger: self.logger.log("[Tracking] Missing P_mats (need >=2 cameras calibrated)")
+            return
+
+        if self.running:
+            return
+        self.running = True
+        self.timer.start(self.tick_ms)
+        if self.logger: self.logger.log("[Tracking] Started")
+
+    def stop(self):
+        if not self.running:
+            return
+        self.running = False
+        self.timer.stop()
+        if self.logger: self.logger.log("[Tracking] Stopped")
+
+    def _tick(self):
+        hands2d = self._detectHands_all_cameras()
+        if not hands2d:
+            return
+
+        # Group hands across cameras (you already have _allocateDetectedHands logic)
+        hand_groups = self._allocateDetectedHands(hands2d)
+
+        # Triangulate (your existing triangulation approach)
+        hands3d = self._triangulateHands(hand_groups)
+
+        if len(hands3d) > 0:
+            self.udp.send_hands3d(hands3d)
+
+        print(
+            f"hands2d={len(hands2d)} groups={len(hand_groups)} hands3d={len(hands3d)}",
+            flush=True
+        )
+
+        if len(hands2d) > 0:
+            print("cam_ids:", sorted(set(h.camera_id for h in hands2d)), flush=True)
+
+    def _detectHands_all_cameras(self):
+        out = []
+        for camera in self.cameraList:
+            ok, frameBGR = camera.capture.read()
+            if not ok or frameBGR is None:
+                continue
+
+            h, w = frameBGR.shape[:2]
+            frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
+            frameRGB.flags.writeable = False
+
+            hand_tracking_result = self.hands.process(frameRGB)
+            if not hand_tracking_result.multi_hand_landmarks:
+                continue
+
+            handed = hand_tracking_result.multi_handedness
+            for hi, lm_list in enumerate(hand_tracking_result.multi_hand_landmarks):
+                landmarks_px = np.array(
+                    [[lm.x * w, lm.y * h] for lm in lm_list.landmark],
+                    dtype=np.float32
+                )
+
+                label, score = "Unknown", 0.0
+                if hi < len(handed) and handed[hi].classification:
+                    label = handed[hi].classification[0].label
+                    score = handed[hi].classification[0].score
+                
+                out.append(Hand2D(
+                    camera_id=camera.index,
+                    handedness=label,
+                    score=score,
+                    landmarks_px=landmarks_px
+                ))
+        return out
+
+    def _allocateDetectedHands(self, hands2d):
+        frames_by_camera = defaultdict(list)
+        for hand in hands2d:
+            frames_by_camera[hand.camera_id].append(hand)
+
+        used = set()
+        hand_groups = []
+        reproj_threshold = 50.0
+
+        camera_ids = sorted(frames_by_camera.keys())
+
+        for cam_a in camera_ids:
+            if cam_a not in self.P_mats:
+                continue
+            for hand_a in frames_by_camera[cam_a]:
+                if id(hand_a) in used:
+                    continue
+
+                group = {cam_a: hand_a}
+                used.add(id(hand_a))
+
+                P_a = self.P_mats[cam_a]
+
+                for cam_b in camera_ids:
+                    if cam_b == cam_a:
+                        continue
+                    if cam_b not in self.P_mats:
+                        continue
+
+                    P_b = self.P_mats[cam_b]
+                    best_hand = None
+                    best_err = np.inf
+
+                    for hand_b in frames_by_camera[cam_b]:
+                        if id(hand_b) in used:
+                            continue
+
+                        err = self.pair_reprojection_error(
+                            hand_a, hand_b, P_a, P_b, key_idxs=[0, 9]
+                        )
+                        if err < best_err:
+                            best_err = err
+                            best_hand = hand_b
+
+                    if best_hand is not None and best_err < reproj_threshold:
+                        group[cam_b] = best_hand
+                        used.add(id(best_hand))
+
+                hand_groups.append(group)
+
+        return hand_groups
+
+    def triangulate_hands_with_metadata(self, hand_groups):
+        hands3d = []
+        for group in hand_groups:
+            if len(group) < 2:
+                continue
+
+            handedness, confidence = self.choose_metadata_from_group(group)
+
+            cams = list(group.keys())
+            P1 = self.P_mats[cams[0]]
+            P2 = self.P_mats[cams[1]]
+            lm1 = group[cams[0]].landmarks_px
+            lm2 = group[cams[1]].landmarks_px
+
+            pts3d = []
+            for i in range(lm1.shape[0]):
+                X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
+                pts3d.append(X)
+
+            hands3d.append({
+                "handedness": handedness,          # "Left"/"Right"/"Unknown"
+                "confidence": confidence,          # 0..1
+                "landmarks": np.asarray(pts3d, dtype=np.float32)  # (21,3)
+            })
+        return hands3d
+
+    # --- math helpers (copy from your existing code) ---
+    def triangulate_point(self, P1, P2, x1, x2):
+        A = np.zeros((4, 4), dtype=np.float32)
+        A[0] = x1[0] * P1[2] - P1[0]
+        A[1] = x1[1] * P1[2] - P1[1]
+        A[2] = x2[0] * P2[2] - P2[0]
+        A[3] = x2[1] * P2[2] - P2[1]
+        _, _, Vt = np.linalg.svd(A)
+        X_h = Vt[-1]
+        X_h /= X_h[3]
+        
+        return X_h[:3]
+
+    def project_point(self, P, X):
+        X_h = np.array([X[0], X[1], X[2], 1.0], dtype=np.float32)
+        x = P @ X_h
+        return np.array([x[0] / x[2], x[1] / x[2]], dtype=np.float32)
+
+    def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs):
+        errors = []
+        for idx in key_idxs:
+            x1 = hand_a.landmarks_px[idx]
+            x2 = hand_b.landmarks_px[idx]
+            X = self.triangulate_point(P_a, P_b, x1, x2)
+            x1_hat = self.project_point(P_a, X)
+            x2_hat = self.project_point(P_b, X)
+            errors.append(np.linalg.norm(x1_hat - x1))
+            errors.append(np.linalg.norm(x2_hat - x2))
+        return float(np.mean(errors))
+
+    def choose_metadata_from_group(group: dict):
+        # group: {cam_id: Hand2D, ...}
+        best = max(group.values(), key=lambda h: float(getattr(h, "score", 0.0)))
+        handedness = getattr(best, "handedness", "Unknown")
+        confidence = float(getattr(best, "score", 0.0))
+        return handedness, confidence
+    
+    def _triangulateHands(self, hand_groups):
+        hands3d = []
+
+        for group in hand_groups:
+            cams = list(group.keys())
+            if len(cams) < 2:
+                continue
+
+            # Pick metadata from best 2D view
+            best = max(group.values(), key=lambda hh: float(getattr(hh, "score", 0.0)))
+            handedness = getattr(best, "handedness", "Unknown")
+            confidence = float(getattr(best, "score", 0.0))
+
+            P1 = self.P_mats[cams[0]]
+            P2 = self.P_mats[cams[1]]
+            lm1 = group[cams[0]].landmarks_px
+            lm2 = group[cams[1]].landmarks_px
+
+            pts3d = []
+            for i in range(lm1.shape[0]):
+                X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
+                pts3d.append(X)
+
+            hands3d.append({
+                "landmarks": np.asarray(pts3d, dtype=np.float32),
+                "handedness": handedness,
+                "confidence": confidence,
+            })
+
+        return hands3d