diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..2c87a1f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run main.py", + "type": "python", + "request": "launch", + "program": "${workspaceFolder}/source/main.py", + "console": "integratedTerminal" + } + ] +} \ No newline at end of file diff --git a/Blender/4017_2025012893110_3030x100.STEP.stl b/Blender/4017_2025012893110_3030x100.STEP.stl new file mode 100644 index 0000000..27610a3 Binary files /dev/null and b/Blender/4017_2025012893110_3030x100.STEP.stl differ diff --git a/Blender/Beamer_Stuff.blend b/Blender/Beamer_Stuff.blend new file mode 100644 index 0000000..50e5a5b Binary files /dev/null and b/Blender/Beamer_Stuff.blend differ diff --git a/Blender/Beamer_Stuff.blend1 b/Blender/Beamer_Stuff.blend1 new file mode 100644 index 0000000..e694f59 Binary files /dev/null and b/Blender/Beamer_Stuff.blend1 differ diff --git a/Blender/Beamer_Stuff.stl b/Blender/Beamer_Stuff.stl new file mode 100644 index 0000000..703feeb Binary files /dev/null and b/Blender/Beamer_Stuff.stl differ diff --git a/Blender/Beamer_StuffCube.stl b/Blender/Beamer_StuffCube.stl new file mode 100644 index 0000000..043fbab Binary files /dev/null and b/Blender/Beamer_StuffCube.stl differ diff --git a/Blender/Beamer_StuffPlane.stl b/Blender/Beamer_StuffPlane.stl new file mode 100644 index 0000000..6d637c5 Binary files /dev/null and b/Blender/Beamer_StuffPlane.stl differ diff --git a/Table.blend b/Blender/Table.blend similarity index 100% rename from Table.blend rename to Blender/Table.blend diff --git a/Table.blend1 b/Blender/Table.blend1 similarity index 100% rename from Table.blend1 rename to Blender/Table.blend1 diff --git a/Table1.blend b/Blender/Table1.blend similarity index 100% rename from Table1.blend rename to Blender/Table1.blend diff --git a/Table1.blend1 b/Blender/Table1.blend1 similarity index 100% rename from Table1.blend1 rename to Blender/Table1.blend1 diff --git a/Blender/Table_Addon_CableHolder.blend b/Blender/Table_Addon_CableHolder.blend new file mode 100644 index 0000000..e448b8f Binary files /dev/null and b/Blender/Table_Addon_CableHolder.blend differ diff --git a/Blender/Table_Addon_CableHolder.blend1 b/Blender/Table_Addon_CableHolder.blend1 new file mode 100644 index 0000000..6da31a3 Binary files /dev/null and b/Blender/Table_Addon_CableHolder.blend1 differ diff --git a/Blender/Table_Addon_CableHolder.stl b/Blender/Table_Addon_CableHolder.stl new file mode 100644 index 0000000..6ccc660 Binary files /dev/null and b/Blender/Table_Addon_CableHolder.stl differ diff --git a/Blender/Table_Addon_USBCamera_Case.blend b/Blender/Table_Addon_USBCamera_Case.blend new file mode 100644 index 0000000..454c43e Binary files /dev/null and b/Blender/Table_Addon_USBCamera_Case.blend differ diff --git a/Blender/Table_Addon_USBCamera_Case.blend1 b/Blender/Table_Addon_USBCamera_Case.blend1 new file mode 100644 index 0000000..3e93302 Binary files /dev/null and b/Blender/Table_Addon_USBCamera_Case.blend1 differ diff --git a/Blender/Table_Addon_USBCamera_Case.stl b/Blender/Table_Addon_USBCamera_Case.stl new file mode 100644 index 0000000..31fc4ee Binary files /dev/null and b/Blender/Table_Addon_USBCamera_Case.stl differ diff --git a/Blender/Table_Addon_USBCamera_CaseBack.stl b/Blender/Table_Addon_USBCamera_CaseBack.stl new file mode 100644 index 0000000..0bccd35 Binary files /dev/null and b/Blender/Table_Addon_USBCamera_CaseBack.stl differ diff --git a/Blender/Table_Addon_USBCamera_CaseClamp.stl b/Blender/Table_Addon_USBCamera_CaseClamp.stl new file mode 100644 index 0000000..d952dbb Binary files /dev/null and b/Blender/Table_Addon_USBCamera_CaseClamp.stl differ diff --git a/Blender/Table_Addon_USBCamera_CaseElgatoStuffDeskMount.stl b/Blender/Table_Addon_USBCamera_CaseElgatoStuffDeskMount.stl new file mode 100644 index 0000000..94329c1 Binary files /dev/null and b/Blender/Table_Addon_USBCamera_CaseElgatoStuffDeskMount.stl differ diff --git a/Blender/Table_Addon_USBCamera_CaseFront.stl b/Blender/Table_Addon_USBCamera_CaseFront.stl new file mode 100644 index 0000000..a6c6e3f Binary files /dev/null and b/Blender/Table_Addon_USBCamera_CaseFront.stl differ diff --git a/Blender/Table_Hooks.blend b/Blender/Table_Hooks.blend new file mode 100644 index 0000000..0ca4424 Binary files /dev/null and b/Blender/Table_Hooks.blend differ diff --git a/Blender/Table_Hooks.blend1 b/Blender/Table_Hooks.blend1 new file mode 100644 index 0000000..e6fa45b Binary files /dev/null and b/Blender/Table_Hooks.blend1 differ diff --git a/Table_Marker.blend b/Blender/Table_Marker.blend similarity index 100% rename from Table_Marker.blend rename to Blender/Table_Marker.blend diff --git a/Blender/Table_Switch_Case.blend b/Blender/Table_Switch_Case.blend new file mode 100644 index 0000000..4e5998a Binary files /dev/null and b/Blender/Table_Switch_Case.blend differ diff --git a/Blender/Table_Switch_Case.blend1 b/Blender/Table_Switch_Case.blend1 new file mode 100644 index 0000000..13a612d Binary files /dev/null and b/Blender/Table_Switch_Case.blend1 differ diff --git a/Table_edge.3mf b/Blender/Table_corner.3mf similarity index 100% rename from Table_edge.3mf rename to Blender/Table_corner.3mf diff --git a/Table_edge.blend b/Blender/Table_corner.blend similarity index 100% rename from Table_edge.blend rename to Blender/Table_corner.blend diff --git a/Table_edge.blend1 b/Blender/Table_corner.blend1 similarity index 100% rename from Table_edge.blend1 rename to Blender/Table_corner.blend1 diff --git a/Table_edge.stl b/Blender/Table_corner.stl similarity index 100% rename from Table_edge.stl rename to Blender/Table_corner.stl diff --git a/Table_edge1.stl b/Blender/Table_corner1.stl similarity index 100% rename from Table_edge1.stl rename to Blender/Table_corner1.stl diff --git a/Blender/Table_edge2.stl b/Blender/Table_edge2.stl new file mode 100644 index 0000000..4582dc3 Binary files /dev/null and b/Blender/Table_edge2.stl differ diff --git a/Blender/Table_edge3.stl b/Blender/Table_edge3.stl new file mode 100644 index 0000000..da1054c Binary files /dev/null and b/Blender/Table_edge3.stl differ diff --git a/Blender/Table_rand.blend b/Blender/Table_rand.blend new file mode 100644 index 0000000..5fca6d5 Binary files /dev/null and b/Blender/Table_rand.blend differ diff --git a/Blender/Table_rand.blend1 b/Blender/Table_rand.blend1 new file mode 100644 index 0000000..0a302a9 Binary files /dev/null and b/Blender/Table_rand.blend1 differ diff --git a/Blender/Table_randPlane.001.3mf b/Blender/Table_randPlane.001.3mf new file mode 100644 index 0000000..c30112b Binary files /dev/null and b/Blender/Table_randPlane.001.3mf differ diff --git a/Blender/Table_randPlane.001.stl b/Blender/Table_randPlane.001.stl new file mode 100644 index 0000000..66ff4c1 Binary files /dev/null and b/Blender/Table_randPlane.001.stl differ diff --git a/Blender/Table_randPlane.002.stl b/Blender/Table_randPlane.002.stl new file mode 100644 index 0000000..26c0106 Binary files /dev/null and b/Blender/Table_randPlane.002.stl differ diff --git a/Blender/Table_randPlane.003.stl b/Blender/Table_randPlane.003.stl new file mode 100644 index 0000000..8dbef17 Binary files /dev/null and b/Blender/Table_randPlane.003.stl differ diff --git a/Blender/Table_randPlane.004.stl b/Blender/Table_randPlane.004.stl new file mode 100644 index 0000000..81fd075 Binary files /dev/null and b/Blender/Table_randPlane.004.stl differ diff --git a/Blender/Table_randPlane.005.stl b/Blender/Table_randPlane.005.stl new file mode 100644 index 0000000..3058d5a Binary files /dev/null and b/Blender/Table_randPlane.005.stl differ diff --git a/Blender/Table_randPlane.006.stl b/Blender/Table_randPlane.006.stl new file mode 100644 index 0000000..6d8d4a6 Binary files /dev/null and b/Blender/Table_randPlane.006.stl differ diff --git a/Blender/Table_randPlane.007.stl b/Blender/Table_randPlane.007.stl new file mode 100644 index 0000000..6442b6a Binary files /dev/null and b/Blender/Table_randPlane.007.stl differ diff --git a/Blender/Table_randPlane.008.stl b/Blender/Table_randPlane.008.stl new file mode 100644 index 0000000..9946c94 Binary files /dev/null and b/Blender/Table_randPlane.008.stl differ diff --git a/Blender/Table_randPlane.009.stl b/Blender/Table_randPlane.009.stl new file mode 100644 index 0000000..1fa0fce Binary files /dev/null and b/Blender/Table_randPlane.009.stl differ diff --git a/Blender/USBHUB.blend b/Blender/USBHUB.blend new file mode 100644 index 0000000..5d2a830 Binary files /dev/null and b/Blender/USBHUB.blend differ diff --git a/Blender/USBHUB.blend1 b/Blender/USBHUB.blend1 new file mode 100644 index 0000000..5438e2b Binary files /dev/null and b/Blender/USBHUB.blend1 differ diff --git a/Blender/USBHUB.stl b/Blender/USBHUB.stl new file mode 100644 index 0000000..364e03a Binary files /dev/null and b/Blender/USBHUB.stl differ diff --git a/Blender/USBHUB3030铝型材理线夹.001.3mf b/Blender/USBHUB3030铝型材理线夹.001.3mf new file mode 100644 index 0000000..a235976 Binary files /dev/null and b/Blender/USBHUB3030铝型材理线夹.001.3mf differ diff --git a/Blender/USBHUB3030铝型材理线夹.001.stl b/Blender/USBHUB3030铝型材理线夹.001.stl new file mode 100644 index 0000000..733f8b4 Binary files /dev/null and b/Blender/USBHUB3030铝型材理线夹.001.stl differ diff --git a/Blender/USBHUBCube.001.stl b/Blender/USBHUBCube.001.stl new file mode 100644 index 0000000..d48270f Binary files /dev/null and b/Blender/USBHUBCube.001.stl differ diff --git a/DarkAngles.pdf b/Stuff/DarkAngles.pdf similarity index 100% rename from DarkAngles.pdf rename to Stuff/DarkAngles.pdf diff --git a/Stuff/Untitled-1.txt b/Stuff/Untitled-1.txt index 7227d5a..608c710 100644 --- a/Stuff/Untitled-1.txt +++ b/Stuff/Untitled-1.txt @@ -1,15 +1,16 @@ -import cv2, numpy as np, time, fitz -from calibration import object_calibration, draw_cal_circle +import cv2, numpy as np +from calibration import object_calibration from hand_detection import detect_hands, check_hand from helper import compute_distance -from hud import find_hud_placement, draw_hud +from source.UI.hud import find_hud_placement import math -from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import FRAME_INTERVAL, TARGET_FPS from config import DEBUG - -from hud import draw_hud_box, draw_pdf_page - +from setup import InitCameras +#from hud import draw_hud_box, draw_pdf_page +import time # ----- Global State ----- measuring = False # measurement in progress @@ -18,27 +19,22 @@ calibrating = False # pinch-based calibration flag object_calibrating = False # object calibration flag fingertip_idx_global = None # last detected fingertip position - hud_pos = None hud_rot = None register_hud = False # ----- Per-camera Processing ----- -def process_frame(frame): +def process_frame(frame, camera_id: int = 0): global measuring, start_pt, calibrating global object_calibrating global fingertip_idx_global global register_hud, hud_pos, hud_rot - frame_out = cv2.flip(frame, 1) - - frame_vis = frame_out.copy() - projection_debug = frame_out.copy() + frame_vis = cv2.flip(frame, 1) # Hand detection - - results = detect_hands(frame_out) + results = detect_hands(frame_vis) fingertip_idx = None if results.multi_hand_landmarks: @@ -52,83 +48,74 @@ def process_frame(frame): object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD) object_calibrating = False - + # place hud res = None if hud_pos is None: res = find_hud_placement(frame_vis) else: res = hud_pos, hud_rot - if res is not None: - arrow_tip, dir_vec = res - draw_hud(frame_vis, arrow_tip, dir_vec) + # create proj output + projection_out = np.zeros_like(frame) + + # measuring + if measuring and start_pt and fingertip_idx_global: + cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2) + px = compute_distance(start_pt, fingertip_idx_global) + inch = px/PIXELS_PER_INCH; cm = inch*2.54 + mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) + cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2) + + # hud + if res is not None or hud_pos: + if hud_pos: + arrow_tip = hud_pos + dir_vec = hud_rot + else: + arrow_tip, dir_vec = res dx, dy = dir_vec arrow_angle = math.atan2(dy, dx) # result in radians - quad = draw_hud_box(frame_vis, arrow_tip, arrow_angle, size=(400,400)) - draw_pdf_page(frame_vis, page_index=0, dst_quad=quad) + #quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400)) + #draw_pdf_page(projection_out, page_index=0, dst_quad=quad) if register_hud: hud_pos = arrow_tip hud_rot = dir_vec register_hud = False - return frame_vis + + mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0 + frame_vis[mask] = projection_out[mask] + + return frame_vis, projection_out # ----- Main ----- ------------------------------------------------------------------------------------- def main(): - global register_hud, hud_pos, hud_rot - - all_captures = [] - for idx in CAMERA_INDICES: - capture = cv2.VideoCapture(idx) - capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) - capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) - all_captures.append(capture) - if not all(capture.isOpened() for capture in all_captures): - print("Error: could not open all cameras") - return - + all_captures = InitCameras() + + print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)") print("Press 'o' for circle calib, 'q' to quit.") - while True: - frames = [capture.read()[1] for capture in all_captures] - frame = next((f for f in frames if f is not None), None) - if frame is None: + last_time = time.time() + while True: + processed_frames = [] + + for camera_id, capture in enumerate(all_captures): + ok, frame = capture.read() + if not ok or frame is None: + continue + + projection, debug = process_frame(frame, camera_id) + processed_frames.append((camera_id, projection, debug)) + + if not processed_frames: break - # process - full_view = process_frame(frame) - - # create proj output - projection_out = np.zeros_like(full_view) - - if measuring and start_pt and fingertip_idx_global: - cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2) - px = compute_distance(start_pt, fingertip_idx_global) - inch = px/PIXELS_PER_INCH; cm = inch*2.54 - mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) - cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2) - - if hud_pos: - dx, dy = hud_rot - arrow_angle = math.atan2(dy, dx) # result in radians - quad = draw_hud_box(projection_out, hud_pos, arrow_angle, size=(400,400)) - draw_pdf_page(projection_out, page_index=0, dst_quad=quad) - - # overlay proj onto debug - debug = full_view.copy() - - # overlay measurement - if measuring and start_pt and fingertip_idx_global: - cv2.line(debug, start_pt, fingertip_idx_global, (255,255,0),2) - cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) - - # show windows - cv2.imshow('Debug Output', debug) - cv2.imshow('Projector Output', projection_out) + for camera_id, projection, debug in processed_frames: + if DEBUG: + cv2.imshow(f'Debug Output (cam {camera_id})', debug) + cv2.imshow(f'Projector Output (cam {camera_id})', projection) key = cv2.waitKey(1) & 0xFF @@ -139,13 +126,22 @@ def main(): object_calibrating = True print("Entered circle calibration mode.") elif key == ord('j'): + global register_hud, hud_pos register_hud = True hud_pos = None + # ——— frame‐rate limiting ——— + now = time.time() + elapsed = now - last_time + to_wait = FRAME_INTERVAL - elapsed + if to_wait > 0: + time.sleep(to_wait) + last_time = time.time() + for capture in all_captures: capture.release() cv2.destroyAllWindows() if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/Table/.obsidian/app.json b/Table/.obsidian/app.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/Table/.obsidian/app.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Table/.obsidian/appearance.json b/Table/.obsidian/appearance.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/Table/.obsidian/appearance.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/Table/.obsidian/core-plugins.json b/Table/.obsidian/core-plugins.json new file mode 100644 index 0000000..0faa60d --- /dev/null +++ b/Table/.obsidian/core-plugins.json @@ -0,0 +1,33 @@ +{ + "file-explorer": true, + "global-search": true, + "switcher": true, + "graph": true, + "backlink": true, + "canvas": true, + "outgoing-link": true, + "tag-pane": true, + "footnotes": false, + "properties": false, + "page-preview": true, + "daily-notes": true, + "templates": true, + "note-composer": true, + "command-palette": true, + "slash-command": false, + "editor-status": true, + "bookmarks": true, + "markdown-importer": false, + "zk-prefixer": false, + "random-note": false, + "outline": true, + "word-count": true, + "slides": false, + "audio-recorder": false, + "workspaces": false, + "file-recovery": true, + "publish": false, + "sync": true, + "bases": true, + "webviewer": false +} \ No newline at end of file diff --git a/Table/.obsidian/graph.json b/Table/.obsidian/graph.json new file mode 100644 index 0000000..42a46ec --- /dev/null +++ b/Table/.obsidian/graph.json @@ -0,0 +1,22 @@ +{ + "collapse-filter": true, + "search": "", + "showTags": false, + "showAttachments": false, + "hideUnresolved": false, + "showOrphans": true, + "collapse-color-groups": true, + "colorGroups": [], + "collapse-display": true, + "showArrow": false, + "textFadeMultiplier": 0, + "nodeSizeMultiplier": 1, + "lineSizeMultiplier": 1, + "collapse-forces": true, + "centerStrength": 0.518713248970312, + "repelStrength": 10, + "linkStrength": 1, + "linkDistance": 250, + "scale": 1, + "close": true +} \ No newline at end of file diff --git a/Table/.obsidian/workspace.json b/Table/.obsidian/workspace.json new file mode 100644 index 0000000..d509508 --- /dev/null +++ b/Table/.obsidian/workspace.json @@ -0,0 +1,254 @@ +{ + "main": { + "id": "833985cf0bef0982", + "type": "split", + "children": [ + { + "id": "9d07647e9b05c8a5", + "type": "tabs", + "dimension": 50, + "children": [ + { + "id": "777c49eacd333b57", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Table/Hand Tracking.md", + "mode": "source", + "source": false + }, + "icon": "lucide-file", + "title": "Hand Tracking" + } + } + ] + }, + { + "id": "ec54e744a9373b63", + "type": "tabs", + "dimension": 50, + "children": [ + { + "id": "55f84d8c4a6e6b37", + "type": "leaf", + "state": { + "type": "graph", + "state": {}, + "icon": "lucide-git-fork", + "title": "Graph view" + } + }, + { + "id": "cdcef64cb6388441", + "type": "leaf", + "state": { + "type": "release-notes", + "state": { + "currentVersion": "1.10.6" + }, + "icon": "lucide-book-up", + "title": "Release Notes 1.10.6" + } + }, + { + "id": "48db892d028c7efb", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Table/Addons/USB Camera Case.md", + "mode": "source", + "source": false + }, + "icon": "lucide-file", + "title": "USB Camera Case" + } + }, + { + "id": "a871b6f7e2ef2b0b", + "type": "leaf", + "state": { + "type": "markdown", + "state": { + "file": "Table/General.md", + "mode": "source", + "source": false + }, + "icon": "lucide-file", + "title": "General" + } + }, + { + "id": "ec002b0aa870cc5d", + "type": "leaf", + "state": { + "type": "release-notes", + "state": { + "currentVersion": "1.11.7" + }, + "icon": "lucide-book-up", + "title": "Release Notes 1.11.7" + } + } + ], + "currentTab": 4 + } + ], + "direction": "vertical" + }, + "left": { + "id": "f6a806b37d1ae6eb", + "type": "split", + "children": [ + { + "id": "9148cc17fa29c8df", + "type": "tabs", + "children": [ + { + "id": "1c44bf34acded4f0", + "type": "leaf", + "state": { + "type": "file-explorer", + "state": { + "sortOrder": "alphabetical", + "autoReveal": false + }, + "icon": "lucide-folder-closed", + "title": "Files" + } + }, + { + "id": "85e41cb90d92a857", + "type": "leaf", + "state": { + "type": "search", + "state": { + "query": "", + "matchingCase": false, + "explainSearch": false, + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical" + }, + "icon": "lucide-search", + "title": "Search" + } + }, + { + "id": "8ada34bb1b02dff6", + "type": "leaf", + "state": { + "type": "bookmarks", + "state": {}, + "icon": "lucide-bookmark", + "title": "Bookmarks" + } + } + ] + } + ], + "direction": "horizontal", + "width": 300 + }, + "right": { + "id": "b343b576de17e5fa", + "type": "split", + "children": [ + { + "id": "a960a5dad3743e2e", + "type": "tabs", + "children": [ + { + "id": "d06b72b904a43a4e", + "type": "leaf", + "state": { + "type": "backlink", + "state": { + "collapseAll": false, + "extraContext": false, + "sortOrder": "alphabetical", + "showSearch": false, + "searchQuery": "", + "backlinkCollapsed": false, + "unlinkedCollapsed": true + }, + "icon": "links-coming-in", + "title": "Backlinks" + } + }, + { + "id": "906e21e00a9b4d73", + "type": "leaf", + "state": { + "type": "outgoing-link", + "state": { + "linksCollapsed": false, + "unlinkedCollapsed": true + }, + "icon": "links-going-out", + "title": "Outgoing links" + } + }, + { + "id": "6c98ae458072df7c", + "type": "leaf", + "state": { + "type": "tag", + "state": { + "sortOrder": "frequency", + "useHierarchy": true, + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-tags", + "title": "Tags" + } + }, + { + "id": "0ce33e15635006fc", + "type": "leaf", + "state": { + "type": "outline", + "state": { + "followCursor": false, + "showSearch": false, + "searchQuery": "" + }, + "icon": "lucide-list", + "title": "Outline" + } + } + ] + } + ], + "direction": "horizontal", + "width": 300, + "collapsed": true + }, + "left-ribbon": { + "hiddenItems": { + "switcher:Open quick switcher": false, + "graph:Open graph view": false, + "canvas:Create new canvas": false, + "daily-notes:Open today's daily note": false, + "templates:Insert template": false, + "command-palette:Open command palette": false, + "bases:Create new base": false + } + }, + "active": "ec002b0aa870cc5d", + "lastOpenFiles": [ + "Table/Hand Tracking.md", + "Table/Addons/Addons.md", + "Table/Addons/Cable Holder.md", + "Table/Addons/Switch.md", + "Table/Addons/USB Camera Case.md", + "Table/Addons/Universal Mount.md", + "Untitled", + "Table/General.md", + "Table/Addons", + "Table/Addons", + "Table" + ] +} \ No newline at end of file diff --git a/Table/Table/Addons/Addons.md b/Table/Table/Addons/Addons.md new file mode 100644 index 0000000..f20857a --- /dev/null +++ b/Table/Table/Addons/Addons.md @@ -0,0 +1,6 @@ + +| Addon | Description | +| ---------------- | ----------------------- | +| [[Switch]] | A casing for the switch | +| [[Cable Holder]] | | + diff --git a/__init__.py b/Table/Table/Addons/Cable Holder.md similarity index 100% rename from __init__.py rename to Table/Table/Addons/Cable Holder.md diff --git a/Table/Table/Addons/Switch.md b/Table/Table/Addons/Switch.md new file mode 100644 index 0000000..4c07be5 --- /dev/null +++ b/Table/Table/Addons/Switch.md @@ -0,0 +1,9 @@ + +| Depth | 11 | +| ----- | ---- | +| Widht | 9.45 | +| Hight | 2.9 | +| | | +| | | +| | | +| | | diff --git a/camera.py b/Table/Table/Addons/USB Camera Case.md similarity index 100% rename from camera.py rename to Table/Table/Addons/USB Camera Case.md diff --git a/Table/Table/Addons/Universal Mount.md b/Table/Table/Addons/Universal Mount.md new file mode 100644 index 0000000..e69de29 diff --git a/Table/Table/General.md b/Table/Table/General.md new file mode 100644 index 0000000..c414776 --- /dev/null +++ b/Table/Table/General.md @@ -0,0 +1,7 @@ +Gaming Table for Warhammer and Stuff. + +[[Addons]] + + + +[[Hand Tracking]] \ No newline at end of file diff --git a/Table/Table/Hand Tracking.md b/Table/Table/Hand Tracking.md new file mode 100644 index 0000000..e69de29 diff --git a/Test/Gestures/gesture_config.json b/Test/Gestures/gesture_config.json new file mode 100644 index 0000000..e69de29 diff --git a/Test/HUDs/DarkAngles/DarkAngles.json b/Test/HUDs/DarkAngles/DarkAngles.json new file mode 100644 index 0000000..1c49440 --- /dev/null +++ b/Test/HUDs/DarkAngles/DarkAngles.json @@ -0,0 +1,9 @@ +{ + "type": "PDFContainer", + "sources": [ + { + "contenttype": "PDF", + "source": "warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf" + } + ] +} \ No newline at end of file diff --git a/Test/HUDs/DarkAngles/FuckThisShit.txt b/Test/HUDs/DarkAngles/FuckThisShit.txt new file mode 100644 index 0000000..c49d6dc --- /dev/null +++ b/Test/HUDs/DarkAngles/FuckThisShit.txt @@ -0,0 +1 @@ +FuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txt \ No newline at end of file diff --git a/Test/HUDs/DarkAngles/Unbenannt.png b/Test/HUDs/DarkAngles/Unbenannt.png new file mode 100644 index 0000000..7ee3b66 Binary files /dev/null and b/Test/HUDs/DarkAngles/Unbenannt.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf new file mode 100644 index 0000000..1e695df Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.zip b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.zip new file mode 100644 index 0000000..f227fcd Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.zip differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_1.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_1.png new file mode 100644 index 0000000..ba9f86c Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_1.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_2.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_2.png new file mode 100644 index 0000000..3dceab0 Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_2.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_3.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_3.png new file mode 100644 index 0000000..d3e8c07 Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk/1776608095514-4325e0a1-f73d-45ba-84f2-b2159e318c7209-s2afk26smk_3.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_1.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_1.png new file mode 100644 index 0000000..ba9f86c Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_1.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_2.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_2.png new file mode 100644 index 0000000..3dceab0 Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_2.png differ diff --git a/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_3.png b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_3.png new file mode 100644 index 0000000..d3e8c07 Binary files /dev/null and b/Test/HUDs/DarkAngles/warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk_3.png differ diff --git a/Test/HUDs/Impreaial_Knights/Impreaial_Knights.json b/Test/HUDs/Impreaial_Knights/Impreaial_Knights.json new file mode 100644 index 0000000..e69de29 diff --git a/Test/HUDs/Krieg/Krieg.json b/Test/HUDs/Krieg/Krieg.json new file mode 100644 index 0000000..e69de29 diff --git a/Test/HUDs/Tyranids/Tyranids.json b/Test/HUDs/Tyranids/Tyranids.json new file mode 100644 index 0000000..e69de29 diff --git a/Test/test.json b/Test/test.json new file mode 100644 index 0000000..63b143c --- /dev/null +++ b/Test/test.json @@ -0,0 +1,163 @@ +{ + "version": 1, + "cameras": [ + { + "index": 0, + "pxWidth": 2048, + "pxHeight": 1536, + "camera_matrix": [ + [ + 1637.301826012801, + 0.0, + 1115.6922895709363 + ], + [ + 0.0, + 1637.3016923198538, + 660.417851928144 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "distortion_coefficients": [ + [ + 0.1880726328811075, + -0.3711284739403174, + -0.019390902558209716, + 0.009835323140375036, + 0.42533165931899103 + ] + ], + "rotation_matrix_world_to_camera": [ + [ + 1.0, + 0.0, + 0.0 + ], + [ + 0.0, + 1.0, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "translation_vector_world_to_camera": [ + [ + 0.0 + ], + [ + 0.0 + ], + [ + 0.0 + ] + ], + "camera_projection_matrix": [ + [ + 1637.301826012801, + 0.0, + 1115.6922895709363, + 0.0 + ], + [ + 0.0, + 1637.3016923198538, + 660.417851928144, + 0.0 + ], + [ + 0.0, + 0.0, + 1.0, + 0.0 + ] + ] + }, + { + "index": 3, + "pxWidth": 2048, + "pxHeight": 1536, + "camera_matrix": [ + [ + 1629.3953951276987, + 0.0, + 946.5260213900043 + ], + [ + 0.0, + 1608.3790414523598, + 656.5060407239171 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "distortion_coefficients": [ + [ + 0.19251720938106318, + -0.21522629593789278, + -0.02730155124857322, + -0.03890715094594556, + 0.21680710398239586 + ] + ], + "rotation_matrix_world_to_camera": [ + [ + 0.08276206632981453, + -0.8544623995610863, + 0.5128785900319194 + ], + [ + 0.720047607579738, + 0.40706667659536877, + 0.5619859105211598 + ], + [ + -0.6889716127646442, + 0.3227858865283794, + 0.6489432858598454 + ] + ], + "translation_vector_world_to_camera": [ + [ + -176.11272583763233 + ], + [ + -564.9970879948509 + ], + [ + 471.9913794371197 + ] + ], + "camera_projection_matrix": [ + [ + -517.2774297117204, + -1086.7318582180455, + 1449.923719330272, + 159794.8580057718 + ], + [ + 705.7954552118755, + 866.628395475963, + 1329.9215473279708, + -598864.2830424493 + ], + [ + -0.6889716127646442, + 0.3227858865283794, + 0.6489432858598454, + 471.9913794371197 + ] + ] + } + ] +} \ No newline at end of file diff --git a/Test/udp_test.py b/Test/udp_test.py new file mode 100644 index 0000000..192b86c --- /dev/null +++ b/Test/udp_test.py @@ -0,0 +1,8 @@ +import socket, struct, time + +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.sendto( + struct.pack("<4sBIQB", b"HAND", 1, 123, int(time.time()*1000), 0), + ("127.0.0.1", 9000) +) +print("sent") \ No newline at end of file diff --git a/charuco_A4.png b/charuco_A4.png new file mode 100644 index 0000000..ada4b1b Binary files /dev/null and b/charuco_A4.png differ diff --git a/helper.py b/helper.py deleted file mode 100644 index 02dc0fa..0000000 --- a/helper.py +++ /dev/null @@ -1,21 +0,0 @@ -import numpy as np -import cv2 - -# ----- Helper Functions ----- -def compute_distance(p1, p2): - return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) - -# 4) compute straightness per finger -def finger_straight(i_tip, i_pip, i_mcp): - d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip))) - d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp))) - return float(np.clip(d1/(d2+1e-3), 0, 1)) - -def yes_no(): - while True: - key = cv2.waitKey(1) & 0xFF - - if key == ord("y"): - return True - if key == ord("n"): - return False diff --git a/hud.py b/hud.py deleted file mode 100644 index 5fc6cce..0000000 --- a/hud.py +++ /dev/null @@ -1,61 +0,0 @@ -import cv2, numpy as np -from config import LOWER_SHAPE, UPPER_SHAPE -import cv2 -import numpy as np -import time -import fitz # PyMuPDF for PDF rendering - - - -def find_hud_placement(frame_vis): - # ----- Arrow Shape Detection & HUD Placement ----- - # 1) Build a clean orange mask - hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE) - kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5)) - mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern) - mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern) - - # 2) Find and filter contours - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - for cnt in contours: - area = cv2.contourArea(cnt) - if area < 1000: - continue - - # Approximate to polygon and require exactly 5 corners - peri = cv2.arcLength(cnt, True) - approx = cv2.approxPolyDP(cnt, 0.02 * peri, True) - if len(approx) != 5: - continue - - pts = approx.reshape(-1,2) - centroid = np.mean(pts, axis=0) - - # Find the arrow tip as the corner farthest from centroid - dists = [np.linalg.norm(pt - centroid) for pt in pts] - tip_pt = pts[int(np.argmax(dists))] - arrow_tip = (int(tip_pt[0]), int(tip_pt[1])) - - # Place the HUD box in the arrow’s pointing direction - dir_vec = tip_pt - centroid - - #ToDo remove - cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2) - - return arrow_tip, dir_vec - -# hud.py -class HUD: - def __init__(self, marker_id, pdf_pages, default_page=0): - self.id = marker_id - self.page_index = default_page - self.pdf_pages = pdf_pages # list of preloaded page images - self.ref_quad = None # the 4-corner quad where to draw the PDF - self.last_seen = 0 # for timeout if arrow goes away - - def update(self, marker): - return - - def draw(self, frame): - return \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index 1ab5fa3..0000000 --- a/main.py +++ /dev/null @@ -1,147 +0,0 @@ -import cv2, numpy as np -from calibration import object_calibration -from hand_detection import detect_hands, check_hand -from helper import compute_distance -from hud import find_hud_placement -import math - -from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD -from config import FRAME_INTERVAL, TARGET_FPS -from config import DEBUG -from setup import InitCameras -#from hud import draw_hud_box, draw_pdf_page -import time - -# ----- Global State ----- -measuring = False # measurement in progress -start_pt = None # measurement start point -calibrating = False # pinch-based calibration flag -object_calibrating = False # object calibration flag -fingertip_idx_global = None # last detected fingertip position - -hud_pos = None -hud_rot = None -register_hud = False - -# ----- Per-camera Processing ----- -def process_frame(frame, camera_id: int = 0): - global measuring, start_pt, calibrating - global object_calibrating - global fingertip_idx_global - - global register_hud, hud_pos, hud_rot - - frame_vis = cv2.flip(frame, 1) - - # Hand detection - results = detect_hands(frame_vis) - - fingertip_idx = None - if results.multi_hand_landmarks: - hand = results.multi_hand_landmarks[0] - fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating) - - fingertip_idx_global = fingertip_idx - - # object calibration - if object_calibrating and fingertip_idx is not None: - object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD) - object_calibrating = False - - # place hud - res = None - if hud_pos is None: - res = find_hud_placement(frame_vis) - else: - res = hud_pos, hud_rot - - # create proj output - projection_out = np.zeros_like(frame) - - # measuring - if measuring and start_pt and fingertip_idx_global: - cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2) - px = compute_distance(start_pt, fingertip_idx_global) - inch = px/PIXELS_PER_INCH; cm = inch*2.54 - mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) - cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2) - - # hud - if res is not None or hud_pos: - if hud_pos: - arrow_tip = hud_pos - dir_vec = hud_rot - else: - arrow_tip, dir_vec = res - - dx, dy = dir_vec - arrow_angle = math.atan2(dy, dx) # result in radians - #quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400)) - #draw_pdf_page(projection_out, page_index=0, dst_quad=quad) - - if register_hud: - hud_pos = arrow_tip - hud_rot = dir_vec - register_hud = False - - - mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0 - frame_vis[mask] = projection_out[mask] - - return frame_vis, projection_out - -# ----- Main ----- ------------------------------------------------------------------------------------- -def main(): - all_captures = InitCameras() - - print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)") - print("Press 'o' for circle calib, 'q' to quit.") - - last_time = time.time() - while True: - processed_frames = [] - - for camera_id, capture in enumerate(all_captures): - ok, frame = capture.read() - if not ok or frame is None: - continue - - projection, debug = process_frame(frame, camera_id) - processed_frames.append((camera_id, projection, debug)) - - if not processed_frames: - break - - for camera_id, projection, debug in processed_frames: - if DEBUG: - cv2.imshow(f'Debug Output (cam {camera_id})', debug) - cv2.imshow(f'Projector Output (cam {camera_id})', projection) - - key = cv2.waitKey(1) & 0xFF - - if key == ord('q'): - break - elif key == ord('o'): - global object_calibrating - object_calibrating = True - print("Entered circle calibration mode.") - elif key == ord('j'): - global register_hud, hud_pos - register_hud = True - hud_pos = None - - # ——— frame‐rate limiting ——— - now = time.time() - elapsed = now - last_time - to_wait = FRAME_INTERVAL - elapsed - if to_wait > 0: - time.sleep(to_wait) - last_time = time.time() - - for capture in all_captures: - capture.release() - - cv2.destroyAllWindows() - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py deleted file mode 100644 index e5d74e6..0000000 --- a/setup.py +++ /dev/null @@ -1,81 +0,0 @@ -import cv2 -import numpy as np - -from config import CAMERA_INDICES_TO_CHECK, FRAME_WIDTH, FRAME_HEIGHT - -def _preview_camera(capture: cv2.VideoCapture, cameraID: int) -> str: - """Show a live preview and return the user's choice: 'add', 'skip', or 'quit'.""" - window_name = f"Camera {cameraID}" - cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) - print(f"[Camera {cameraID}] Press 'y' to add, 'n' to skip, 'q' to stop scanning.") - - try: - while True: - ok, frame = capture.read() - if not ok: - print(f"[Camera {cameraID}] Failed to read frame; skipping.") - return "skip" - - cv2.imshow(window_name, frame) - - key = cv2.waitKey(1) & 0xFF - - if key == ord("y"): - print(f"[Camera {cameraID}] Selected.") - return "add" - if key == ord("n"): - print(f"[Camera {cameraID}] Skipped.") - return "skip" - if key == ord("q") or key == 27: # 27 == ESC - print(f"[Camera {cameraID}] Stopping camera scan.") - return "quit" - finally: - cv2.destroyWindow(window_name) - - -def InitCameras(): - allCaptures = [] - - setup_window = "Camera Setup" - instructions = np.zeros((240, 560, 3), dtype=np.uint8) - cv2.putText(instructions, "Camera Setup", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 200, 255), 2) - cv2.putText(instructions, "Each camera will preview in its own window.", (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) - cv2.putText(instructions, "Use 'y' to add, 'n' to skip, 'q'/ESC to finish.", (20, 125), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) - cv2.putText(instructions, "Close this window when you're done.", (20, 160), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) - - cv2.namedWindow(setup_window, cv2.WINDOW_AUTOSIZE) - cv2.imshow(setup_window, instructions) - cv2.waitKey(1) - - try: - for cameraID in range(CAMERA_INDICES_TO_CHECK): - capture = cv2.VideoCapture(cameraID) - - if not capture.isOpened(): - capture.release() - continue - - capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) - capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) - - decision = _preview_camera(capture, cameraID) - - if decision == "add": - allCaptures.append(capture) - else: - capture.release() - - if decision == "quit": - break - - if not all(capture.isOpened() for capture in allCaptures): - print("Error: could not open all selected cameras.") - return - - if not allCaptures: - print("No cameras selected.") - return - - return allCaptures - finally: - cv2.destroyWindow(setup_window) diff --git a/source/Helpers/CHArUco_version_test.py b/source/Helpers/CHArUco_version_test.py new file mode 100644 index 0000000..f084b74 --- /dev/null +++ b/source/Helpers/CHArUco_version_test.py @@ -0,0 +1,5 @@ +import cv2 +print("OpenCV:", cv2.__version__) +print("has aruco:", hasattr(cv2, "aruco")) +print("has CharucoBoard:", hasattr(cv2.aruco, "CharucoBoard")) +print("has ArucoDetector:", hasattr(cv2.aruco, "ArucoDetector")) \ No newline at end of file diff --git a/source/Helpers/ChArUco.py b/source/Helpers/ChArUco.py new file mode 100644 index 0000000..11cda97 --- /dev/null +++ b/source/Helpers/ChArUco.py @@ -0,0 +1,32 @@ +import cv2 +import numpy as np + +# ---- Board parameters (CHANGE THESE ONLY IF YOU REPRINT) ---- +squares_x = 6 # number of chessboard squares in X +squares_y = 9 # number of chessboard squares in Y +square_length = 25 # mm +marker_length = 18 # mm (must be < square_length) +dictionary_id = cv2.aruco.DICT_4X4_50 +dpi = 300 # print DPI +# ------------------------------------------------------------ + +dictionary = cv2.aruco.getPredefinedDictionary(dictionary_id) +board = cv2.aruco.CharucoBoard( + (squares_x, squares_y), + square_length, + marker_length, + dictionary +) + +# Convert physical size (mm) to pixels for printing +mm_to_inch = 1 / 25.4 +width_mm = squares_x * square_length +height_mm = squares_y * square_length + +width_px = int(width_mm * mm_to_inch * dpi) +height_px = int(height_mm * mm_to_inch * dpi) + +img = board.generateImage((width_px, height_px)) +img = 255 - img +cv2.imwrite("H:\Table\charuco_A4.png", img) +print("Saved charuco_A4.png") \ No newline at end of file diff --git a/source/Helpers/ChArUco_confidence_Test.py b/source/Helpers/ChArUco_confidence_Test.py new file mode 100644 index 0000000..f82fed7 --- /dev/null +++ b/source/Helpers/ChArUco_confidence_Test.py @@ -0,0 +1,220 @@ +""" +ChArUco detection confidence test (OpenCV) + +What it does: +- Opens one or multiple cameras +- Detects ArUco markers and interpolated ChArUco corners +- Draws overlays +- Prints a simple "confidence" score per camera: + markers_found, charuco_corners_found, and a normalized confidence value + +Requirements: +- OpenCV built with aruco module (opencv-contrib-python) + pip install opencv-contrib-python +""" + +import cv2 +import numpy as np +import time + +# ---------------------------- +# USER SETTINGS (match your printed board!) +# ---------------------------- +CAM_IDS = [0, 2] # set to [0] for single camera test, or [0,2,3,...] +USE_DSHOW_ON_WINDOWS = True # good for Windows +RESOLUTION = (3264, 2448) # (width, height) if your cameras support it; else set None +DICT_ID = cv2.aruco.DICT_4X4_50 + +SQUARES_X = 6 # number of chessboard squares in X +SQUARES_Y = 9 # number of chessboard squares in Y +SQUARE_LEN_MM = 25.0 # square size in mm +MARKER_LEN_MM = 18.0 # marker size in mm (must be < square) + +# Confidence thresholds (tune if needed) +MIN_MARKERS_OK = 4 # markers to consider "good" +MIN_CHARUCO_OK = 15 # charuco corners to consider "good" + +# Display +WINDOW_SCALE = 0.5 # downscale for display if res is huge (0.5 = half size) +PRINT_EVERY_SEC = 0.5 +# ---------------------------- + + +def open_camera(cam_id: int): + backend = cv2.CAP_DSHOW if (USE_DSHOW_ON_WINDOWS and hasattr(cv2, "CAP_DSHOW")) else 0 + cap = cv2.VideoCapture(cam_id, backend) + if not cap.isOpened(): + return None + + if RESOLUTION is not None: + w, h = RESOLUTION + cap.set(cv2.CAP_PROP_FRAME_WIDTH, float(w)) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, float(h)) + + return cap + + +def compute_confidence(markers: int, charuco: int) -> float: + """ + Simple normalized confidence heuristic: + - markers contribute up to MIN_MARKERS_OK + - charuco corners contribute up to MIN_CHARUCO_OK + """ + m = min(markers / max(MIN_MARKERS_OK, 1), 1.0) + c = min(charuco / max(MIN_CHARUCO_OK, 1), 1.0) + # Weighted: charuco corners matter more for calibration quality + return 0.35 * m + 0.65 * c + + +def main(): + dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID) + board = cv2.aruco.CharucoBoard( + (SQUARES_X, SQUARES_Y), + SQUARE_LEN_MM, + MARKER_LEN_MM, + dictionary + ) + + # Detector parameters + detector_params = cv2.aruco.DetectorParameters() + # You can tweak these if detection is unstable: + # detector_params.adaptiveThreshWinSizeMin = 3 + # detector_params.adaptiveThreshWinSizeMax = 23 + # detector_params.adaptiveThreshWinSizeStep = 10 + + detector = cv2.aruco.ArucoDetector(dictionary, detector_params) + + caps = {} + for cam_id in CAM_IDS: + cap = open_camera(cam_id) + if cap is None: + print(f"[ERR] Could not open camera {cam_id}") + else: + caps[cam_id] = cap + print(f"[OK] Opened camera {cam_id}") + + if not caps: + print("No cameras opened. Exiting.") + return + + last_print = 0.0 + + print("\nControls:") + print(" ESC = quit") + print(" Space = print one-shot stats immediately\n") + + try: + while True: + frames_vis = [] + stats = {} + + for cam_id, cap in caps.items(): + ok, frame = cap.read() + if not ok or frame is None: + stats[cam_id] = (0, 0, 0.0) + continue + + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + gray = 255 - gray + # Detect markers + corners, ids, rejected = detector.detectMarkers(gray) + markers_found = 0 if ids is None else len(ids) + + # Draw markers + vis = frame.copy() + if ids is not None: + cv2.aruco.drawDetectedMarkers(vis, corners, ids) + + # Interpolate ChArUco corners (requires some markers found) + charuco_found = 0 + if ids is not None and len(ids) > 0: + charuco_detector = cv2.aruco.CharucoDetector(board) + + charuco_corners = None + charuco_ids = None + + if ids is not None and len(ids) > 0: + res = charuco_detector.detectBoard(gray) + + # OpenCV versions differ in what they return; handle both safely. + # Common patterns: + # (charucoCorners, charucoIds, markerCorners, markerIds) + # (charucoCorners, charucoIds, rejectedMarkerCandidates) + # (charucoCorners, charucoIds, ...) + charuco_corners = res[0] if len(res) > 0 else None + charuco_ids = res[1] if len(res) > 1 else None + charuco_found = 0 + if charuco_ids is not None: + charuco_found = len(charuco_ids) + cv2.aruco.drawDetectedCornersCharuco(vis, charuco_corners, charuco_ids) + + conf = compute_confidence(markers_found, charuco_found) + stats[cam_id] = (markers_found, charuco_found, conf) + + # Overlay text + h, w = vis.shape[:2] + lines = [ + f"Cam {cam_id}", + f"Markers: {markers_found}", + f"ChArUco corners: {charuco_found}", + f"Confidence: {conf:.2f}", + "GOOD" if (markers_found >= MIN_MARKERS_OK and charuco_found >= MIN_CHARUCO_OK) else "MOVE / LIGHT / FOCUS" + ] + + y = 30 + for line in lines: + cv2.putText(vis, line, (20, y), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0) if "GOOD" in line else (0, 200, 255), 2) + y += 32 + + # Downscale for display + if WINDOW_SCALE != 1.0: + vis = cv2.resize(vis, None, fx=WINDOW_SCALE, fy=WINDOW_SCALE, interpolation=cv2.INTER_AREA) + + frames_vis.append(vis) + + # Combine displays + if frames_vis: + # Stack horizontally; if many cams, wrap to multiple rows + max_per_row = 3 + rows = [] + for i in range(0, len(frames_vis), max_per_row): + row = frames_vis[i:i + max_per_row] + # pad heights + max_h = max(img.shape[0] for img in row) + padded = [] + for img in row: + if img.shape[0] < max_h: + pad = max_h - img.shape[0] + img = cv2.copyMakeBorder(img, 0, pad, 0, 0, cv2.BORDER_CONSTANT, value=(0, 0, 0)) + padded.append(img) + rows.append(np.hstack(padded)) + grid = np.vstack(rows) + + cv2.imshow("ChArUco Confidence Test", grid) + + key = cv2.waitKey(1) & 0xFF + now = time.time() + + if key == 27: # ESC + break + + if key == 32: # Space + last_print = 0 # force print now + + if now - last_print >= PRINT_EVERY_SEC: + last_print = now + # Print compact stats + msg = " | ".join( + f"cam{cid}: M={m} C={c} conf={conf:.2f}" + for cid, (m, c, conf) in sorted(stats.items()) + ) + print(msg) + + finally: + for cap in caps.values(): + cap.release() + cv2.destroyAllWindows() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/source/Helpers/HandUdpSender.py b/source/Helpers/HandUdpSender.py new file mode 100644 index 0000000..c3bed80 --- /dev/null +++ b/source/Helpers/HandUdpSender.py @@ -0,0 +1,57 @@ +import socket +import struct +import time +import numpy as np + +class HandUdpSender: + MAGIC = b'HAND' + VERSION = 2 + + def __init__(self, host="127.0.0.1", port=9000): + self.addr = (host, port) + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.seq = 0 + + def _encode_handedness(self, s: str) -> int: + if s == "Left": + return 1 + if s == "Right": + return 2 + return 0 + + def send_hands3d(self, hands3d: list[dict]): + """ + hands3d: list of dicts: + { + "landmarks": (21,3) np.ndarray float, + "handedness": "Left"/"Right"/"Unknown", + "confidence": float (0..1) + } + Sends one UDP datagram containing all hands. + """ + self.seq += 1 + ts_ms = int(time.time() * 1000) + + hand_count = min(len(hands3d), 255) + + # Header: magic(4s), version(B), seq(I), ts(Q), hand_count(B) + packet = bytearray() + packet += struct.pack("<4sBIQB", self.MAGIC, self.VERSION, self.seq, ts_ms, hand_count) + + for hid in range(hand_count): + h = hands3d[hid] + pts = np.asarray(h["landmarks"], dtype=np.float32) + if pts.shape != (21, 3): + continue + + point_count = 21 + handedness_code = self._encode_handedness(str(h.get("handedness", "Unknown"))) + confidence = float(h.get("confidence", 0.0)) + + # per-hand header: hand_id(B), point_count(B), handedness(B), confidence(f) + packet += struct.pack("=2 cameras calibrated)") + return + + if self.running: + return + self.running = True + self.timer.start(self.tick_ms) + if self.logger: self.logger.log("[Tracking] Started") + + def stop(self): + if not self.running: + return + self.running = False + self.timer.stop() + if self.logger: self.logger.log("[Tracking] Stopped") + + def _tick(self): + hands2d = self._detectHands_all_cameras() + if not hands2d: + return + + # Group hands across cameras (you already have _allocateDetectedHands logic) + hand_groups = self._allocateDetectedHands(hands2d) + + # Triangulate (your existing triangulation approach) + hands3d = self._triangulateHands(hand_groups) + + if len(hands3d) > 0: + self.udp.send_hands3d(hands3d) + + print( + f"hands2d={len(hands2d)} groups={len(hand_groups)} hands3d={len(hands3d)}", + flush=True + ) + + if len(hands2d) > 0: + print("cam_ids:", sorted(set(h.camera_id for h in hands2d)), flush=True) + + def _detectHands_all_cameras(self): + out = [] + for camera in self.cameraList: + ok, frameBGR = camera.capture.read() + if not ok or frameBGR is None: + continue + + h, w = frameBGR.shape[:2] + frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB) + frameRGB.flags.writeable = False + + hand_tracking_result = self.hands.process(frameRGB) + if not hand_tracking_result.multi_hand_landmarks: + continue + + handed = hand_tracking_result.multi_handedness + for hi, lm_list in enumerate(hand_tracking_result.multi_hand_landmarks): + landmarks_px = np.array( + [[lm.x * w, lm.y * h] for lm in lm_list.landmark], + dtype=np.float32 + ) + + label, score = "Unknown", 0.0 + if hi < len(handed) and handed[hi].classification: + label = handed[hi].classification[0].label + score = handed[hi].classification[0].score + + out.append(Hand2D( + camera_id=camera.index, + handedness=label, + score=score, + landmarks_px=landmarks_px + )) + return out + + def _allocateDetectedHands(self, hands2d): + frames_by_camera = defaultdict(list) + for hand in hands2d: + frames_by_camera[hand.camera_id].append(hand) + + used = set() + hand_groups = [] + reproj_threshold = 50.0 + + camera_ids = sorted(frames_by_camera.keys()) + + for cam_a in camera_ids: + if cam_a not in self.P_mats: + continue + for hand_a in frames_by_camera[cam_a]: + if id(hand_a) in used: + continue + + group = {cam_a: hand_a} + used.add(id(hand_a)) + + P_a = self.P_mats[cam_a] + + for cam_b in camera_ids: + if cam_b == cam_a: + continue + if cam_b not in self.P_mats: + continue + + P_b = self.P_mats[cam_b] + best_hand = None + best_err = np.inf + + for hand_b in frames_by_camera[cam_b]: + if id(hand_b) in used: + continue + + err = self.pair_reprojection_error( + hand_a, hand_b, P_a, P_b, key_idxs=[0, 9] + ) + if err < best_err: + best_err = err + best_hand = hand_b + + if best_hand is not None and best_err < reproj_threshold: + group[cam_b] = best_hand + used.add(id(best_hand)) + + hand_groups.append(group) + + return hand_groups + + def triangulate_hands_with_metadata(self, hand_groups): + hands3d = [] + for group in hand_groups: + if len(group) < 2: + continue + + handedness, confidence = self.choose_metadata_from_group(group) + + cams = list(group.keys()) + P1 = self.P_mats[cams[0]] + P2 = self.P_mats[cams[1]] + lm1 = group[cams[0]].landmarks_px + lm2 = group[cams[1]].landmarks_px + + pts3d = [] + for i in range(lm1.shape[0]): + X = self.triangulate_point(P1, P2, lm1[i], lm2[i]) + pts3d.append(X) + + hands3d.append({ + "handedness": handedness, # "Left"/"Right"/"Unknown" + "confidence": confidence, # 0..1 + "landmarks": np.asarray(pts3d, dtype=np.float32) # (21,3) + }) + return hands3d + + # --- math helpers (copy from your existing code) --- + def triangulate_point(self, P1, P2, x1, x2): + A = np.zeros((4, 4), dtype=np.float32) + A[0] = x1[0] * P1[2] - P1[0] + A[1] = x1[1] * P1[2] - P1[1] + A[2] = x2[0] * P2[2] - P2[0] + A[3] = x2[1] * P2[2] - P2[1] + _, _, Vt = np.linalg.svd(A) + X_h = Vt[-1] + X_h /= X_h[3] + + return X_h[:3] + + def project_point(self, P, X): + X_h = np.array([X[0], X[1], X[2], 1.0], dtype=np.float32) + x = P @ X_h + return np.array([x[0] / x[2], x[1] / x[2]], dtype=np.float32) + + def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs): + errors = [] + for idx in key_idxs: + x1 = hand_a.landmarks_px[idx] + x2 = hand_b.landmarks_px[idx] + X = self.triangulate_point(P_a, P_b, x1, x2) + x1_hat = self.project_point(P_a, X) + x2_hat = self.project_point(P_b, X) + errors.append(np.linalg.norm(x1_hat - x1)) + errors.append(np.linalg.norm(x2_hat - x2)) + return float(np.mean(errors)) + + def choose_metadata_from_group(group: dict): + # group: {cam_id: Hand2D, ...} + best = max(group.values(), key=lambda h: float(getattr(h, "score", 0.0))) + handedness = getattr(best, "handedness", "Unknown") + confidence = float(getattr(best, "score", 0.0)) + return handedness, confidence + + def _triangulateHands(self, hand_groups): + hands3d = [] + + for group in hand_groups: + cams = list(group.keys()) + if len(cams) < 2: + continue + + # Pick metadata from best 2D view + best = max(group.values(), key=lambda hh: float(getattr(hh, "score", 0.0))) + handedness = getattr(best, "handedness", "Unknown") + confidence = float(getattr(best, "score", 0.0)) + + P1 = self.P_mats[cams[0]] + P2 = self.P_mats[cams[1]] + lm1 = group[cams[0]].landmarks_px + lm2 = group[cams[1]].landmarks_px + + pts3d = [] + for i in range(lm1.shape[0]): + X = self.triangulate_point(P1, P2, lm1[i], lm2[i]) + pts3d.append(X) + + hands3d.append({ + "landmarks": np.asarray(pts3d, dtype=np.float32), + "handedness": handedness, + "confidence": confidence, + }) + + return hands3d \ No newline at end of file diff --git a/source/Tbd/__pycache__/TrackingController.cpython-311.pyc b/source/Tbd/__pycache__/TrackingController.cpython-311.pyc new file mode 100644 index 0000000..1519ae1 Binary files /dev/null and b/source/Tbd/__pycache__/TrackingController.cpython-311.pyc differ diff --git a/source/Tbd/__pycache__/helper.cpython-311.pyc b/source/Tbd/__pycache__/helper.cpython-311.pyc new file mode 100644 index 0000000..c8a4d8b Binary files /dev/null and b/source/Tbd/__pycache__/helper.cpython-311.pyc differ diff --git a/calibration.py b/source/Tbd/calibration.py similarity index 100% rename from calibration.py rename to source/Tbd/calibration.py diff --git a/source/Tbd/camera.py b/source/Tbd/camera.py new file mode 100644 index 0000000..e69de29 diff --git a/config.py b/source/Tbd/config.py similarity index 100% rename from config.py rename to source/Tbd/config.py diff --git a/hand_detection.py b/source/Tbd/hand_detection.py similarity index 100% rename from hand_detection.py rename to source/Tbd/hand_detection.py diff --git a/source/Tbd/helper.py b/source/Tbd/helper.py new file mode 100644 index 0000000..8140839 --- /dev/null +++ b/source/Tbd/helper.py @@ -0,0 +1,66 @@ +import numpy as np +import json +import cv2 + +from dataclasses import dataclass, field, asdict +from typing import List, Optional, Tuple, Dict + + +# ----- Helper Functions ----- +def compute_distance(p1, p2): + return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) + +# 4) compute straightness per finger +def finger_straight(i_tip, i_pip, i_mcp): + d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip))) + d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp))) + return float(np.clip(d1/(d2+1e-3), 0, 1)) + + +def default_profile(): + return [] + +@dataclass +class CameraObject(): + capture: Optional[cv2.VideoCapture] + index: int + pxHeight: int = 1536 + pxWidth: int = 2048 + + # Calibration + camera_matrix: Optional[np.ndarray] = None # 3x3 + distortion_coefficients: Optional[np.ndarray] = None # 1x5 / 1x8 + rotation_matrix_world_to_camera: Optional[np.ndarray] = None # 3x3 (world->cam) + translation_vector_world_to_camera: Optional[np.ndarray] = None # 3x1 (world->cam) + camera_projection_matrix: Optional[np.ndarray] = None + + def load_video_capture(self): + #self.capture = cv2.VideoCapture(self.index, cv2.CAP_MSMF) + self.capture = cv2.VideoCapture(self.index, cv2.CAP_ANY) + #self.capture = cv2.VideoCapture(self.index, cv2.CAP_DSHOW) + self.capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2048.0) + self.capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1536.0) + + +@dataclass +class Hand2D(): + camera_id: str # e.g. "overhead_left" + handedness: str # "Left" / "Right" + score: float # detection/tracking confidence + landmarks_px: np.ndarray # shape (21,2) in pixels + + +@dataclass +class CharucoDetection: + camera_id: int + + # Output of Charuco detection + charuco_corners: np.ndarray # (N, 1, 2) float32 + charuco_ids: np.ndarray # (N, 1) int32 + + gray_frame: Optional[np.ndarray] = None + + +@dataclass +class CalibrationSample: + detections: dict[int, CharucoDetection] # camera_id → detection \ No newline at end of file diff --git a/Stuff/import cv2.py b/source/Tbd/import cv2.py similarity index 100% rename from Stuff/import cv2.py rename to source/Tbd/import cv2.py diff --git a/source/Tbd/setup.py b/source/Tbd/setup.py new file mode 100644 index 0000000..e69de29 diff --git a/source/UI/CameraSetupWidget.py b/source/UI/CameraSetupWidget.py new file mode 100644 index 0000000..5b68584 --- /dev/null +++ b/source/UI/CameraSetupWidget.py @@ -0,0 +1,67 @@ +import cv2, numpy as np + +from Tbd.helper import CameraObject + +from PySide6.QtCore import Qt, QTimer, QSize +from PySide6.QtGui import QImage, QPixmap + +from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QSizePolicy, QHBoxLayout, QLineEdit, QPushButton +from typing import List +from PySide6.QtWidgets import QScrollArea, QFrame + + +class CameraSetupWidget(QWidget): + + def __init__(self, cameraObject: CameraObject): + super().__init__() + + self.cameraObject = cameraObject + + self.mainLayout = QHBoxLayout(self) + + self.frame_preview_widget = QLabel() + self.preview_max_size = QSize(800, 600) + self.frame_preview_widget.setMaximumSize(self.preview_max_size) + self.camerOptionsLayout = QVBoxLayout() + + self.start_preview_button = QPushButton("Start") + self.stop_preview_button = QPushButton("Stop") + + + + self.camerOptionsLayout.addWidget(self.start_preview_button) + self.camerOptionsLayout.addWidget(self.stop_preview_button) + + self.mainLayout.addLayout(self.camerOptionsLayout) + self.mainLayout.addWidget(self.frame_preview_widget) + + self.start_preview_button.clicked.connect(self._start_camera_preview) + self.stop_preview_button.clicked.connect(self._stop_camera_preview) + + self.frame_update_timer = QTimer(self) + self.frame_update_timer.timeout.connect(self._update_frame) + + + def _start_camera_preview(self): + self.frame_update_timer.start(30) + + def _stop_camera_preview(self): + self.frame_update_timer.stop() + + + + def _update_frame(self): + ok, frame = self.cameraObject.capture.read() + if not ok or frame is None: + return + + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + height, width, ch = frame_rgb.shape + qimg = QImage(frame_rgb.data, width, height, ch * width, QImage.Format_RGB888) + pixmap = QPixmap.fromImage(qimg) + pixmap = pixmap.scaled( + self.frame_preview_widget.maximumSize(), + Qt.KeepAspectRatio, + Qt.SmoothTransformation, + ) + self.frame_preview_widget.setPixmap(pixmap) diff --git a/source/UI/GameWindow.py b/source/UI/GameWindow.py new file mode 100644 index 0000000..4474df7 --- /dev/null +++ b/source/UI/GameWindow.py @@ -0,0 +1,333 @@ +import cv2 +import numpy +import mediapipe +from Tbd.helper import CameraObject, Hand2D +from PySide6.QtCore import Qt, QTimer +from PySide6.QtWidgets import QMainWindow, QLabel, QVBoxLayout, QWidget +from PySide6.QtGui import QImage, QPixmap +from typing import List +from collections import defaultdict +import winsound + +mediapipe_options = mediapipe.solutions.hands + +class GameWindow(QMainWindow): + """ + Separate window for the “game” part. + You can put your rendering, controls, etc. here. + """ + + def __init__(self, cameraList: List[CameraObject], width_px=1920, height_px=1080, fps=30, max_num_hands=4): + super().__init__() + + self.P_mats = {} + + self.cameraList = cameraList + self._hands = mediapipe_options.Hands( + static_image_mode=False, + max_num_hands=max_num_hands, + model_complexity=1, + min_detection_confidence=0.5, + min_tracking_confidence=0.5, + ) + + self.setWindowTitle("Game Window") + self.canvas_w = width_px + self.canvas_h = height_px + + + self.view = QLabel("HUD") + self.view.setAlignment(Qt.AlignCenter) + self.view.setScaledContents(False) # keep aspect ratio + + + central = QWidget(self) + self.setCentralWidget(central) + layout = QVBoxLayout(central) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(self.view, 1) + + # Create a Timer for my _tick function and start it + self.timer = QTimer(self) + self.timer.timeout.connect(self._tick) + self.timer.start(30) # ~33 FPS + + def _tick(self): + + # 1) create a black BGR canvas + frame_bgr = numpy.zeros((self.canvas_h, self.canvas_w, 3), dtype=numpy.uint8) + + # 2) (optional) draw HUD widgets here + cv2.putText(frame_bgr, "HUD ready", (40, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (200,200,200), 2, cv2.LINE_AA) + + hands2d = self._detectHands() + + self.draw_hands_on_frame(frame_bgr, hands2d) + + # 3) draw detected hands (in canvas pixel coords) + for h in hands2d: + for (x, y) in h.landmarks_px.astype(int): + cv2.circle(frame_bgr, (x, y), 4, (0, 255, 255), -1, cv2.LINE_AA) + + # 4) show it + frame_rgb = cv2.cvtColor(frame_bgr, cv2.COLOR_BGR2RGB) + qimg = QImage(frame_rgb.data, self.canvas_w, self.canvas_h, + self.canvas_w * 3, QImage.Format_RGB888) + pix = QPixmap.fromImage(qimg) + # keep aspect ratio when fitting into the label + self.view.setPixmap(pix.scaled(self.view.size(), Qt.KeepAspectRatio, Qt.SmoothTransformation)) + + hand_groups = self._allocateDetectedHands(hands2d) + + hands3d = self._triangulateHands(hand_groups) + print(hands3d) + self._updateGeasture() + self._checkInteractions() + self._upateHUD() + self._drawHUD() + + + # Loop all Cameras and return a List of all found Hands in all Cameras + def _detectHands(self): + out: List[Hand2D] = [] # List of all found Hands + + + for camera in self.cameraList: + + # Get the frame of the camera and check if the camera is available + ok, frameBGR = camera.capture.read() + if not ok or frameBGR is None: + continue + + frameHight, frameWidht = frameBGR.shape[:2] # Take only the the frame hight and widht from the Tuple + + frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB) # Convert to RGB becaus,e MediaPipe expects RGB + frameRGB.flags.writeable = False # (minor speed gain) + + # Detect Hands in the frame and return a list of all found Hands, then check if + foundHands = self._hands.process(frameRGB) + if not foundHands.multi_hand_landmarks: + continue + + # handedness info aligns with landmarks list + handed = foundHands.multi_handedness + + for handIndex, landmarlList in enumerate(foundHands.multi_hand_landmarks): + landmarks_px = numpy.array([[landmark.x*frameWidht, landmark.y*frameHight] for landmark in landmarlList.landmark], dtype=numpy.float32) + + label = "Unknown" + score = 0.0 + + if handIndex < len(handed): + classifications = handed[handIndex].classification + if classifications: + label = classifications[0].label # "Left" / "Right" + score = classifications[0].score # confidence + + out.append(Hand2D( + camera_id=camera.index, + handedness=label, + score=score, + landmarks_px=landmarks_px + )) + + + return out + + def _allocateDetectedHands(self, hands2d: list): + """ + Take flat list of Hand2D from all cameras and group them into physical hands. + Writes self._hand_groups = [ {cam_id: Hand2D, ...}, ... ] + """ + + # Group by camera + frames_by_camera = defaultdict(list) + for hand in hands2d: + frames_by_camera[hand.camera_id].append(hand) + + # We will mark which Hand2D detections are already consumed + used = set() # set of id(hand2d) + hand_groups = [] + + # Threshold in pixels for “same hand” + reproj_threshold = 8.0 # tune this + + camera_ids = sorted(frames_by_camera.keys()) + + for cam_a in camera_ids: + for hand_a in frames_by_camera[cam_a]: + if id(hand_a) in used: + continue + + # Start a new group with this detection as the seed + group = {cam_a: hand_a} + used.add(id(hand_a)) + + P_a = self.P_mats[cam_a] + + # Try to find matching hands in all other cameras + for cam_b in camera_ids: + if cam_b == cam_a: + continue + if cam_b not in self.P_mats: + continue + + P_b = self.P_mats[cam_b] + + best_hand = None + best_err = numpy.inf + + for hand_b in frames_by_camera[cam_b]: + if id(hand_b) in used: + continue + + err = self.pair_reprojection_error( + hand_a, hand_b, + P_a, P_b, + key_idxs=[0, 9] # wrist + middle MCP for example + ) + + if err < best_err: + best_err = err + best_hand = hand_b + + if best_hand is not None and best_err < reproj_threshold: + group[cam_b] = best_hand + used.add(id(best_hand)) + + hand_groups.append(group) + + return hand_groups + + + def _triangulateHands(self, hand_groups): + + hands3d = [] + + for group in hand_groups: + # use all cameras in 'group' to triangulate each landmark + cams = list(group.keys()) + hands = [group[c] for c in cams] + + # Example: simple pairwise triangulation using first two cameras + if len(cams) < 2: + continue # need at least 2 views + + P1 = self.P_mats[cams[0]] + P2 = self.P_mats[cams[1]] + lm1 = hands[0].landmarks_px + lm2 = hands[1].landmarks_px + + pts3d = [] + for i in range(lm1.shape[0]): + X = self.triangulate_point(P1, P2, lm1[i], lm2[i]) + pts3d.append(X) + + pts3d = numpy.array(pts3d, dtype=numpy.float32) + hands3d.append(pts3d) + + return hands3d + + def _updateGeasture(self): + + return + + def _getMostConfindentHand(self): + + return + + def _checkInteractions(self): + + return + + def _upateHUD(self): + + return + + def _drawHUD(self): + + return + + + def close(self): + print("What?") + self.timer.stop() + return super().close() + + + def draw_hands_on_frame(self, frame_bgr: numpy.ndarray, hands2d: list): + # Draw connections first, then points + for h in hands2d: + pts = h.landmarks_px.astype(int) # (21,2) in pixels + + # bones + for a, b in mediapipe_options.HAND_CONNECTIONS: + cv2.line(frame_bgr, + (int(pts[a,0]), int(pts[a,1])), + (int(pts[b,0]), int(pts[b,1])), + (0, 255, 0), 2, cv2.LINE_AA) + + # joints + for (x, y) in pts: + cv2.circle(frame_bgr, (int(x), int(y)), 3, (0, 0, 255), -1, cv2.LINE_AA) + + # optional label + cv2.putText(frame_bgr, f"{h.handedness} {h.score:.2f}", + (int(pts[0,0]), int(pts[0,1])-8), + cv2.FONT_HERSHEY_SIMPLEX, 0.5, (80,255,80), 1, cv2.LINE_AA) + + def project_point(self, P, X): + """ + P: 3x4 projection matrix + X: 3D point (X, Y, Z) + returns: 2D point (u, v) in pixels + """ + X_h = numpy.array([X[0], X[1], X[2], 1.0], dtype=numpy.float32) + x = P @ X_h + return numpy.array([x[0] / x[2], x[1] / x[2]], dtype=numpy.float32) + + def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs): + """ + hand_a, hand_b: Hand2D objects + P_a, P_b: 3x4 projection matrices + key_idxs: list of landmark indices to use (e.g. [0, 9]) + returns: average reprojection error in pixels + """ + errors = [] + + for idx in key_idxs: + x1 = hand_a.landmarks_px[idx] # (u, v) + x2 = hand_b.landmarks_px[idx] + + X = self.triangulate_point(P_a, P_b, x1, x2) + + x1_hat = self.project_point(P_a, X) + x2_hat = self.project_point(P_b, X) + + e1 = numpy.linalg.norm(x1_hat - x1) + e2 = numpy.linalg.norm(x2_hat - x2) + + errors.append(e1) + errors.append(e2) + + return float(numpy.mean(errors)) + + + def triangulate_point(self, P1, P2, x1, x2): + """ + P1, P2: 3x4 projection matrices + x1, x2: 2D points (u, v) in pixels (float) + returns: 3D point in world coords (X, Y, Z) + """ + A = numpy.zeros((4, 4), dtype=numpy.float32) + A[0] = x1[0] * P1[2] - P1[0] + A[1] = x1[1] * P1[2] - P1[1] + A[2] = x2[0] * P2[2] - P2[0] + A[3] = x2[1] * P2[2] - P2[1] + + # Solve A * X = 0, X is homogeneous 4D + _, _, Vt = numpy.linalg.svd(A) + X_h = Vt[-1] + X_h /= X_h[3] + return X_h[:3] # (X, Y, Z) + diff --git a/source/UI/HomePageWidget.py b/source/UI/HomePageWidget.py new file mode 100644 index 0000000..2709dfb --- /dev/null +++ b/source/UI/HomePageWidget.py @@ -0,0 +1,505 @@ +import cv2 +import numpy + +from Tbd.helper import CameraObject, CharucoDetection, CalibrationSample +from UI.CameraSetupWidget import CameraSetupWidget +from Helpers import saveManager as sm + +from PySide6.QtCore import Qt, QTimer +from PySide6.QtGui import QImage, QPixmap + +from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QSizePolicy +from typing import List +from PySide6.QtWidgets import QScrollArea, QFrame, QPushButton + +from UI.UILogger import UILogger + +MIN_CHARUCO_CORNERS = 15 # absolute minimum +#GOOD_CHARUCO_CORNERS = 25 # ideal + +DICT_ID = cv2.aruco.DICT_4X4_50 +SQUARES_X = 6 # number of chessboard squares in X +SQUARES_Y = 9 # number of chessboard squares in Y +SQUARE_LEN_MM = 25.0 # square size in mm +MARKER_LEN_MM = 18.0 # marker size in mm (must be < square) + +#FRAME_SIZE = [3264, 2448] +#FRAME_SIZE = [1536, 2048] +FRAME_SIZE = [2048, 1536] + +REFERENCE_CAMERA_INDEX = 1 + +class HomeWindow(QWidget): + + def __init__(self, cameraList: List[CameraObject], on_cameraList_changed, P_mats, logger : UILogger): + super().__init__() + + # Initiating variables + self.P_mats = P_mats + self.on_cameraList_changed = on_cameraList_changed + self.cameraList = cameraList + self.logger = logger + self.calibration_samples: list[CalibrationSample] = [] + self.camera_Setup_Widget_List = [] + + + # Charcuo detection stuff + dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID) + self.board = cv2.aruco.CharucoBoard( + (SQUARES_X, SQUARES_Y), + SQUARE_LEN_MM, + MARKER_LEN_MM, + dictionary + ) + self.charuco_detector = cv2.aruco.CharucoDetector(self.board) + # Detector parameters + detector_params = cv2.aruco.DetectorParameters() + # You can tweak these if detection is unstable: + # detector_params.adaptiveThreshWinSizeMin = 3 + # detector_params.adaptiveThreshWinSizeMax = 23 + # detector_params.adaptiveThreshWinSizeStep = 10 + + self.arcuo_detector = cv2.aruco.ArucoDetector(dictionary, detector_params) + + # Creating the main_Layout and other Widgets + self.main_Layout = QVBoxLayout(self) + self.scroll_Box = QScrollArea() + self.calibration_button = QPushButton("Calibrate") + self.caputre_calibration_sample_button = QPushButton("Capture calibration sample") + self.clear_calibration_samples_button = QPushButton("Clear calibration samples") + self.load_calibration_button = QPushButton("Load calibration") + self.save_calibration_button = QPushButton("Save calibration") + + + # Creating the Scrollable area + self.scroll_Box.setWidgetResizable(True) + self.scroll_Content = QWidget() + self.scroll_Layout = QVBoxLayout(self.scroll_Content) + self.scroll_Layout.setAlignment(Qt.AlignTop) + self.scroll_Box.setWidget(self.scroll_Content) + + # Adding Primary show elements + self.main_Layout.addWidget(self.clear_calibration_samples_button) + self.main_Layout.addWidget(self.load_calibration_button) + self.main_Layout.addWidget(self.save_calibration_button) + self.main_Layout.addWidget(self.caputre_calibration_sample_button) + self.main_Layout.addWidget(self.calibration_button) + self.main_Layout.addWidget(self.scroll_Box) + + # Connecting Buttons to functions + self.calibration_button.clicked.connect(self._calibrate_camera) + self.caputre_calibration_sample_button.clicked.connect(self._caputre_calibration_sample) + self.clear_calibration_samples_button.clicked.connect(self._clear_calibration_samples) + self.load_calibration_button.clicked.connect(self._load_calibration) + self.save_calibration_button.clicked.connect(self._save_calibration) + + + def _clear_calibration_samples(self): + self.calibration_samples.clear() + self.logger.log("Calibration samples where cleared.") + + def _load_calibration(self): + loaded = sm.load_calibration_json("C:\\git\\Table\\Test\\test.json", self.logger) + self.cameraList.clear() + self.cameraList.extend(loaded) + self.P_mats.clear() + self.P_mats.update({ + camera.index: camera.camera_projection_matrix + for camera in self.cameraList + if camera.camera_projection_matrix is not None + }) + self.on_cameraList_changed() + + def _save_calibration(self): + sm.save_calibration_json(self.cameraList, "C:\\git\\Table\\Test\\test.json", self.logger) + + def _calibrate_camera(self): + self.logger.log("Fuck you") + + if(len(self.cameraList) < 2): + self.logger.log("You need at least 2 cameras for calibration") + return + + self._clear_camera_calibrations() + + REFERENCE_CAMERA_INDEX = self.cameraList[0].index + + for camera in self.cameraList: + reprojection_error_rms, camera_matrix, distortion_coefficients = self._calibrate_intrinsics_characuo_for_camera( + sample_list = self.calibration_samples, + camera_index = camera.index, + board = self.board + ) + + camera.camera_matrix = camera_matrix + camera.distortion_coefficients = distortion_coefficients + + self.logger.log(f"[Intrinsics] cam{camera.index}: RMS={reprojection_error_rms:.4f} fx={camera_matrix[0,0]:.2f} fy={camera_matrix[1,1]:.2f} cx={camera_matrix[0,2]:.2f} cy={camera_matrix[1,2]:.2f}") + + reference_camera = next(camera for camera in self.cameraList if camera.index == REFERENCE_CAMERA_INDEX) + + self.logger.log("Intrinsics done") + + self.logger.log("Start extrinsics1") + # Reference camera projection + reference_camera.rotation_matrix_world_to_camera = numpy.eye(3) + reference_camera.translation_vector_world_to_camera = numpy.zeros((3,1)) + reference_camera.camera_projection_matrix = ( + reference_camera.camera_matrix + @ numpy.hstack([numpy.eye(3), numpy.zeros((3,1))]) + ) + self.logger.log("Start extrinsics2") + self.P_mats.clear() + self.P_mats[reference_camera.index] = reference_camera.camera_projection_matrix + self.logger.log("Start extrinsics3") + for camera_to_calibrate in self.cameraList: + if camera_to_calibrate.index == REFERENCE_CAMERA_INDEX: + continue + + + stereo_rms,R_ref_to_cam,t_ref_to_cam,E, F, P_ref, P_cam = self._stereo_calibrate_from_charuco_samples( + sample_list=self.calibration_samples, + board=self.board, + camera_refernece=reference_camera, + camera_to_calibrate=camera_to_calibrate + ) + + + camera_to_calibrate.rotation_matrix_world_to_camera = R_ref_to_cam + camera_to_calibrate.translation_vector_world_to_camera = t_ref_to_cam + camera_to_calibrate.camera_projection_matrix = P_cam + + self.P_mats[camera_to_calibrate.index] = camera_to_calibrate.camera_projection_matrix + + + self.logger.log("Extrinsics done") + self.logger.log("Amount of P_mats:" + str(len(self.P_mats))) + self.logger.log("Amount of cameras:" + str(len(self.cameraList))) + self.log_all_camera_calibration(self.cameraList) + + + def rotation_matrix_to_rpy_deg(self, R: numpy.ndarray): + """ + Convert rotation matrix to roll, pitch, yaw in degrees. + Assumes right-handed, OpenCV convention. + """ + sy = numpy.sqrt(R[0,0]*R[0,0] + R[1,0]*R[1,0]) + singular = sy < 1e-6 + + if not singular: + roll = numpy.arctan2(R[2,1], R[2,2]) + pitch = numpy.arctan2(-R[2,0], sy) + yaw = numpy.arctan2(R[1,0], R[0,0]) + else: + roll = numpy.arctan2(-R[1,2], R[1,1]) + pitch = numpy.arctan2(-R[2,0], sy) + yaw = 0.0 + + return numpy.degrees([roll, pitch, yaw]) + + def log_all_camera_calibration(self, camera_list): + self.logger.log("========== CAMERA CALIBRATION SUMMARY ==========") + + for cam in camera_list: + self.logger.log(f"--- Camera {cam.index} ---") + + # -------- Intrinsics -------- + if cam.camera_matrix is not None: + K = cam.camera_matrix + fx, fy = K[0,0], K[1,1] + cx, cy = K[0,2], K[1,2] + + self.logger.log( + f"Intrinsics:" + f" fx={fx:.2f}, fy={fy:.2f}," + f" cx={cx:.2f}, cy={cy:.2f}" + ) + + if cam.distortion_coefficients is not None: + d = cam.distortion_coefficients.flatten() + d_short = ", ".join(f"{v:.4f}" for v in d[:5]) + self.logger.log(f"Distortion: [{d_short}{'...' if len(d) > 5 else ''}]") + else: + self.logger.log("Intrinsics: NOT SET") + + # -------- Extrinsics -------- + if cam.rotation_matrix_world_to_camera is not None and cam.translation_vector_world_to_camera is not None: + R = cam.rotation_matrix_world_to_camera + t = cam.translation_vector_world_to_camera.reshape(3) + + roll, pitch, yaw = self.rotation_matrix_to_rpy_deg(R) + dist = numpy.linalg.norm(t) + + self.logger.log( + f"Extrinsics (world → cam):" + f" t=({t[0]:.1f}, {t[1]:.1f}, {t[2]:.1f})" + f" | |t|={dist:.1f}" + ) + self.logger.log( + f"Rotation (deg):" + f" roll={roll:.2f}, pitch={pitch:.2f}, yaw={yaw:.2f}" + ) + else: + self.logger.log("Extrinsics: NOT SET") + + # -------- Projection -------- + if cam.camera_projection_matrix is not None: + P = cam.camera_projection_matrix + self.logger.log(f"Projection matrix: shape={P.shape}") + else: + self.logger.log("Projection matrix: NOT SET") + + self.logger.log("==============================================") + + def _build_stereo_correspondences_from_samples( + self, + sample_list, + board, + camera_index_reference, + camera_index_to_calibrate + ): + object_points_per_frame = [] + frame_points_per_frame_reference = [] + frame_points_per_frame_to_calibrate = [] + + for sample in sample_list: + + # Get detections for both cameras and check if valide + detection_reference = sample.detections.get(camera_index_reference) + detection_to_calibrate = sample.detections.get(camera_index_to_calibrate) + + if detection_reference is None or detection_to_calibrate is None: + continue + + if detection_reference.charuco_ids is None or detection_reference.charuco_corners is None: + continue + + if detection_to_calibrate.charuco_ids is None or detection_to_calibrate.charuco_corners is None: + continue + + # Find corner IDs that both cameras can see and check if the amount is enough + ids_reference = detection_reference.charuco_ids.reshape(-1) + ids_to_calibrate = detection_to_calibrate.charuco_ids.reshape(-1) + + common_ids = numpy.intersect1d(ids_reference, ids_to_calibrate) + + if len(common_ids) < MIN_CHARUCO_CORNERS: + continue + + # Build ordered correspondences by common_ids + # Map id -> corner for each cam + map_a = {int(i): detection_reference.charuco_corners[idx] for idx, i in enumerate(ids_reference)} + map_b = {int(i): detection_to_calibrate.charuco_corners[idx] for idx, i in enumerate(ids_to_calibrate)} + + # Assemble corners/ids arrays in matching order + corners_reference = numpy.array([map_a[int(i)] for i in common_ids], dtype=numpy.float32).reshape(-1, 1, 2) + corners_to_calibrate = numpy.array([map_b[int(i)] for i in common_ids], dtype=numpy.float32).reshape(-1, 1, 2) + ids_common = common_ids.astype(numpy.int32).reshape(-1, 1) + + # Convert ChArUco corners+ids -> (objectPts, imagePts) for the board + object_points_reference, frame_pts_reference = board.matchImagePoints(corners_reference, ids_common) + _, frame_pts_to_calibrate = board.matchImagePoints(corners_to_calibrate, ids_common) + + if object_points_reference is None or frame_pts_reference is None or frame_pts_to_calibrate is None: + continue + + object_points_reference = self._as_np_float32(object_points_reference) + frame_pts_reference = self._as_np_float32(frame_pts_reference) + frame_pts_to_calibrate = self._as_np_float32(frame_pts_to_calibrate) + + if len(object_points_reference) < MIN_CHARUCO_CORNERS: + continue + + object_points_per_frame.append(object_points_reference) + frame_points_per_frame_reference.append(frame_pts_reference) + frame_points_per_frame_to_calibrate.append(frame_pts_to_calibrate) + + return object_points_per_frame, frame_points_per_frame_reference, frame_points_per_frame_to_calibrate + + def _as_np_float32(self, x): + return numpy.asarray(x, dtype=numpy.float32) + + def _stereo_calibrate_from_charuco_samples( + self, + sample_list, + board, + camera_refernece, + camera_to_calibrate + ): + + if camera_refernece.camera_matrix is None or camera_refernece.distortion_coefficients is None: + self.logger.log(f"cam{camera_refernece.index} missing intrinsics") + if camera_to_calibrate.camera_matrix is None or camera_to_calibrate.distortion_coefficients is None: + self.logger.log(f"cam{camera_to_calibrate.index} missing intrinsics") + + object_points_per_frame, frame_points_per_frame_reference, frame_points_per_frame_to_calibrate = self._build_stereo_correspondences_from_samples( + sample_list=sample_list, + board=board, + camera_index_reference=camera_refernece.index, + camera_index_to_calibrate=camera_to_calibrate.index + ) + + frame_width, frame_height = FRAME_SIZE + + # Keep intrinsics fixed (recommended since you already calibrated them) + flags = cv2.CALIB_FIX_INTRINSIC + + # Termination criteria for stereo calibration optimizer + criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 1e-6) + + stereo_rms_reprojection_error_px, camera_matrix_reference, dist_reference, camera_matrix_to_calibrate, dist_to_calibrate, \ + rotation_matrix_reference_to_calibrate, translation_vector_reference_to_calibrate, \ + essential_matrix, fundamental_matrix = cv2.stereoCalibrate( + objectPoints=object_points_per_frame, + imagePoints1=frame_points_per_frame_reference, + imagePoints2=frame_points_per_frame_to_calibrate, + cameraMatrix1=camera_refernece.camera_matrix, + distCoeffs1=camera_refernece.distortion_coefficients, + cameraMatrix2=camera_to_calibrate.camera_matrix, + distCoeffs2=camera_to_calibrate.distortion_coefficients, + imageSize=(frame_width, frame_height), + criteria=criteria, + flags=flags + ) + + projection_matrix_camA = camera_refernece.camera_matrix @ numpy.hstack([numpy.eye(3, dtype=numpy.float64), numpy.zeros((3,1), dtype=numpy.float64)]) + projection_matrix_camB = camera_to_calibrate.camera_matrix @ numpy.hstack([rotation_matrix_reference_to_calibrate, translation_vector_reference_to_calibrate]) + + + + return ( + float(stereo_rms_reprojection_error_px), + rotation_matrix_reference_to_calibrate, + translation_vector_reference_to_calibrate, + essential_matrix, + fundamental_matrix, + projection_matrix_camA, + projection_matrix_camB, + ) + + def _clear_camera_calibrations(self): + for camera in self.cameraList: + camera.camera_matrix = None + camera.distortion_coefficients = None + camera.rotation_matrix_world_to_camera = None + camera.translation_vector_world_to_camera = None + + def _calibrate_intrinsics_characuo_for_camera( + self, + sample_list: CalibrationSample, + camera_index: int, + board + ): + object_points_per_frame = [] + frame_points_per_frame = [] + + for sample in sample_list: + detection = sample.detections.get(camera_index) + + if detection is None: + continue + + if detection.charuco_ids is None or detection.charuco_corners is None: + continue + + if len(detection.charuco_ids) < MIN_CHARUCO_CORNERS: + continue + + # map detected 2D corners + ids to corresponding 3D board points + object_points, frame_points = board.matchImagePoints(detection.charuco_corners, detection.charuco_ids) + + if object_points is None or frame_points is None: + continue + + object_points_per_frame.append(object_points) + frame_points_per_frame.append(frame_points) + + + + frame_width, frame_height = FRAME_SIZE + self.logger.log(f"Calibrating cam{camera_index} with imageSize={FRAME_SIZE} (w,h)") + self.logger.log(f"H= {frame_width} W= {frame_height}") + + reprojection_error_rms, camera_matrix, distortion_coefficients, rvecs, tvecs = cv2.calibrateCamera( + objectPoints=object_points_per_frame, + imagePoints=frame_points_per_frame, + imageSize=(frame_width, frame_height), + cameraMatrix=None, + distCoeffs=None + ) + + return float(reprojection_error_rms), camera_matrix, distortion_coefficients + + def _caputre_calibration_sample(self): + detections = {} + + for camera in self.cameraList: # Loop all Cameras + ok, frame = camera.capture.read() #Capture a frame + + if not ok: # Check if captured frame is valid, if even one frame is invalide the whole capture failed + self.logger.log(f"Could not grab frame for calibration sample, from camera {camera.index}.") + return + + h, w = frame.shape[:2] + self.logger.log(f"cam{camera.index} capture size = {w}x{h}") + + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Turn the frame black and white + gray = 255 - gray # Invert the color of the frame, as the ChArCuo board I use is inverted (to save printer Ink) + + h, w = gray.shape[:2] + self.logger.log(f"cam{camera.index} capture size = {w}x{h}") + + #This is apparently a fast check, bevor i detect the actuall board + corner, ids, _ = self.arcuo_detector.detectMarkers(gray) #Detect ChArCuo Markers + if ids is None or len(ids) == 0: #Check if Markers where found, if none where found the sample is invalid + self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.") + return + + # Detecting the Charcuo board and evaluating if its good as a sample + result = self.charuco_detector.detectBoard(gray) + + charuco_corners = result[0] + charuco_ids = result[1] + + if charuco_ids is None: + self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.") + return + + if len(charuco_ids) < MIN_CHARUCO_CORNERS: + self.logger.log(f"Not enought markers found for calibration sample, from camera {camera.index}.") + return + + # Adding the sample for the current camera to detections + detections[camera.index] = CharucoDetection( + camera_id = camera.index, + charuco_corners = charuco_corners, + charuco_ids=charuco_ids, + gray_frame = gray + ) + + # All cameras found valide markers so the sample capture was succesfull + self.calibration_samples.append( + CalibrationSample(detections=detections) + ) + + self.logger.log("Added calibration sample.") + + def updateCameras(self): + self._clearLayout() + + for camera_Object in self.cameraList: + camera_Setup_Widget = CameraSetupWidget(camera_Object) + + self.scroll_Layout.addWidget(camera_Setup_Widget) + self.camera_Setup_Widget_List.append(camera_Setup_Widget) + + def _clearLayout(self): + self.camera_Setup_Widget_List.clear() + + while self.scroll_Layout.count(): + item = self.scroll_Layout.takeAt(0) + widget = item.widget() + if widget is not None: + widget.setParent(None) + widget.deleteLater() + diff --git a/source/UI/SetupPageWidget.py b/source/UI/SetupPageWidget.py new file mode 100644 index 0000000..0de989e --- /dev/null +++ b/source/UI/SetupPageWidget.py @@ -0,0 +1,228 @@ +import cv2, numpy as np +import os + + +from Tbd.helper import CameraObject +from UI.UILogger import UILogger + +from PySide6.QtCore import Qt, QTimer +from PySide6.QtGui import QImage, QPixmap +from PySide6.QtWidgets import QLineEdit +from PySide6.QtGui import QIntValidator +from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QPushButton, QHBoxLayout +from typing import List + + + + +class SetupPage(QWidget): + """ + Page that handles camera preview & cycling through cameras. + """ + + def __init__(self, cameraList: List[CameraObject], on_cameraList_changed, logger : UILogger): + super().__init__() + + self.on_cameraList_changed = on_cameraList_changed + self.cameraList = cameraList + self.logger = logger + self.cameraObject = CameraObject( + capture=None, + index=None + ) + + # --- UI --- + self.setUpLayout = QVBoxLayout(self) + + self.setUpLayout.addLayout(self._make_Setup_Controls()) + self.setUpLayout.addLayout(self._make_Setup_Options()) + + def _make_Setup_Options(self): + setupOptions = QHBoxLayout() + setupOptions2 = QVBoxLayout() + + self.cameraIndexLable = QLabel("Camera Index:") + self.cameraIndex = QLineEdit("0") + self.cameraIndex.setValidator(QIntValidator(0, 9999, self)) + + self.addCameraBTN = QPushButton("Add Camera") + self.nextCameraBTN = QPushButton("Next Camera") + self.previousCameraBTN = QPushButton("Previous Camera") + + setupOptions.addWidget(self.cameraIndexLable) + setupOptions.addWidget(self.cameraIndex, 1) + setupOptions.addWidget(self.addCameraBTN) + setupOptions.addWidget(self.nextCameraBTN) + setupOptions.addWidget(self.previousCameraBTN) + + self.nextCameraBTN.clicked.connect(self._nextIndex) + self.previousCameraBTN.clicked.connect(self._previousIndex) + #self.cameraIndex.editingFinished.connect(self._updateCamera) + self.addCameraBTN.clicked.connect(self._addCamera) + + # --- + self.camera_preview = QLabel("No camera") + self.camera_preview.setAlignment(Qt.AlignCenter) + self.camera_preview.setMinimumSize(640, 360) + self.camera_preview.setStyleSheet("background: #222; color: #aaa;") + + self.timer = QTimer(self) + self.timer.timeout.connect(self._updateCamera) + + setupOptions2.addLayout(setupOptions) + setupOptions2.addWidget(self.camera_preview) + + + return setupOptions2 + + def _addCamera(self): + + if any(camera.index == int(self.cameraIndex.text()) for camera in self.cameraList): + print("Camera already added.") + return + + self.cameraList.append(self.cameraObject) + self._cameraList_changed() + + + def load_setup(self): + if (os.path.exists(self.setupFilePath.text())): + self.logger.log("Path found") + else: + self.logger.log("Path not found") + + + indexToAdd = [0, 3] + #self.cameraObject = CameraObject( + # capture=None, + # index=None + #) + for index in indexToAdd: + capture = cv2.VideoCapture(index, cv2.CAP_MSMF) + #capture = cv2.VideoCapture(index, cv2.CAP_DSHOW) + + capture.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG')) + capture.set(cv2.CAP_PROP_FPS, 30) + capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2592) + capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1944) + + #capture.set(cv2.CAP_PROP_FRAME_WIDTH, 2048.0) + #capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 1536.0) + width = capture.get(cv2.CAP_PROP_FRAME_WIDTH) + height = capture.get(cv2.CAP_PROP_FRAME_HEIGHT) + fps = capture.get(cv2.CAP_PROP_FPS) + + fourcc = int(capture.get(cv2.CAP_PROP_FOURCC)) + fourcc_str = "".join([chr((fourcc >> 8*i) & 0xFF) for i in range(4)]) + + print(f"Resolution: {width}x{height}") + print(f"FPS: {fps}") + print(f"Format (FOURCC): {fourcc_str}") + self.logger.log(f"Resolution: {width}x{height}") + self.logger.log(f"FPS: {fps}") + self.logger.log(f"Format (FOURCC): {fourcc_str}") + #currentcapture = cv2.VideoCapture(index, cv2.CAP_MSMF) + + #self.logger.log("Initial frame:", w, "x", h) + self.cameraList.append(CameraObject( + capture=capture, + index=index + )) + self._cameraList_changed() + + + return + + + + def save_setup(self): + + print("ToDo") + return True + + def _cameraList_changed(self): + if self.on_cameraList_changed: + self.on_cameraList_changed() + + def _updateCamera(self): + if self.cameraObject.capture is None: + return + ret, frame = self.cameraObject.capture.read() + if not ret: + return + + frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + h, w, ch = frame_rgb.shape + bytes_per_line = ch * w + qimg = QImage(frame_rgb.data, w, h, bytes_per_line, QImage.Format_RGB888) + self.camera_preview.setPixmap(QPixmap.fromImage(qimg)) + + def _nextIndex(self): + cameraIndex = int(self.cameraIndex.text()) + self.cameraIndex.setText(str(cameraIndex + 1)) + + if self.open_camera(): + if not self.timer.isActive(): + self.timer.start(30) + + self._updateCamera() + + def _previousIndex(self): + cameraIndex = int(self.cameraIndex.text()) + + if((cameraIndex - 1) < 0): + self.cameraIndex.setText("0") + + self.cameraIndex.setText(str(cameraIndex - 1)) + + if self.open_camera(): + if not self.timer.isActive(): + self.timer.start(30) + + self._updateCamera() + + def open_camera(self): + index = int(self.cameraIndex.text()) + + self.release_camera() + self.cameraObject.capture = cv2.VideoCapture(index) + self.cameraObject.index = index + + if not self.cameraObject.capture.isOpened(): + self.cameraObject.capture = None + self.cameraObject.index = None + self.camera_preview.setText(f"Failed to open camera {index}") + return False + + self.camera_preview.setText("") + + return True + + def release_camera(self): + if self.cameraObject.capture is not None: + self.timer.stop() + self.cameraObject.capture.release() + self.cameraObject.capture = None + self.cameraObject.index = None + + def closeEvent(self, event): + self.release_camera() + event.accept() + + def _make_Setup_Controls(self): + # Create Setup Input + setupFileLayout = QHBoxLayout() + self.setupFilePathLable = QLabel("Setupfile path:") + self.setupFilePath = QLineEdit() + self.loadBTN = QPushButton("Load") + self.saveBTN = QPushButton("Save") + + setupFileLayout.addWidget(self.setupFilePathLable) + setupFileLayout.addWidget(self.setupFilePath, 1) + setupFileLayout.addWidget(self.loadBTN) + setupFileLayout.addWidget(self.saveBTN) + + self.loadBTN.clicked.connect(self.load_setup) + self.saveBTN.clicked.connect(self.save_setup) + + return setupFileLayout diff --git a/source/UI/UILogger.py b/source/UI/UILogger.py new file mode 100644 index 0000000..1012504 --- /dev/null +++ b/source/UI/UILogger.py @@ -0,0 +1,23 @@ +from PySide6.QtCore import QObject, Signal, Slot +from PySide6.QtWidgets import QPlainTextEdit +import winsound + +class UILogger(QObject): + append_line = Signal(str) + + def __init__(self, widget: QPlainTextEdit): + super().__init__() + + self.widget = widget + self.widget.setReadOnly(True) + self.widget.setMaximumBlockCount(2000) + self.append_line.connect(self._append) + + + @Slot(str) + def _append(self, text: str): + self.widget.appendPlainText(text) + + def log(self, text:str): + self.append_line.emit(text) + winsound.PlaySound("SystemExclamation", winsound.SND_ALIAS | winsound.SND_ASYNC) diff --git a/source/UI/__pycache__/CameraSetupWidget.cpython-311.pyc b/source/UI/__pycache__/CameraSetupWidget.cpython-311.pyc new file mode 100644 index 0000000..79c8714 Binary files /dev/null and b/source/UI/__pycache__/CameraSetupWidget.cpython-311.pyc differ diff --git a/source/UI/__pycache__/GameWindow.cpython-311.pyc b/source/UI/__pycache__/GameWindow.cpython-311.pyc new file mode 100644 index 0000000..7c5fe14 Binary files /dev/null and b/source/UI/__pycache__/GameWindow.cpython-311.pyc differ diff --git a/source/UI/__pycache__/HomePageWidget.cpython-311.pyc b/source/UI/__pycache__/HomePageWidget.cpython-311.pyc new file mode 100644 index 0000000..276a63f Binary files /dev/null and b/source/UI/__pycache__/HomePageWidget.cpython-311.pyc differ diff --git a/source/UI/__pycache__/SetupPageWidget.cpython-311.pyc b/source/UI/__pycache__/SetupPageWidget.cpython-311.pyc new file mode 100644 index 0000000..e55ab62 Binary files /dev/null and b/source/UI/__pycache__/SetupPageWidget.cpython-311.pyc differ diff --git a/source/UI/__pycache__/UILogger.cpython-311.pyc b/source/UI/__pycache__/UILogger.cpython-311.pyc new file mode 100644 index 0000000..9adbd78 Binary files /dev/null and b/source/UI/__pycache__/UILogger.cpython-311.pyc differ diff --git a/source/__init__.py b/source/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/main.py b/source/main.py new file mode 100644 index 0000000..4dd8e51 --- /dev/null +++ b/source/main.py @@ -0,0 +1,164 @@ +import cv2, numpy as np +import sys +import os +import json +from pathlib import Path + +from Tbd.helper import default_profile, CameraObject +from UI.SetupPageWidget import SetupPage +from UI.HomePageWidget import HomeWindow +from UI.GameWindow import GameWindow +from UI.UILogger import UILogger +from Tbd.TrackingController import TrackingController + +from PySide6.QtCore import Qt, QTimer +from PySide6.QtGui import QImage, QPixmap +from PySide6.QtWidgets import ( + QApplication, + QMainWindow, + QStackedWidget, + QMessageBox, + QPlainTextEdit +) +from PySide6.QtWidgets import QLineEdit +from PySide6.QtGui import QIntValidator +from PySide6.QtWidgets import QWidget, QLabel, QVBoxLayout, QPushButton, QHBoxLayout, QSizePolicy +from typing import List + +CONFIG_PATH = Path("config/profile_default.json") + + + +class MainWindow(QMainWindow): + """ + Main control window: left sidebar + right content area. + """ + cameraList = [] + + def __init__(self): + super().__init__() + + self.P_mats = {} + + self.setWindowTitle("Gaming Table Control") + self.resize(1200, 1200) + + # Central root widget + root = QWidget() + self.setCentralWidget(root) + + self.main_layout = QVBoxLayout(root) + self.main_interaction_layout = QHBoxLayout() + + # Create Logger + self.console = QPlainTextEdit() + self.logger = UILogger(self.console) + + # ---------- Sidebar ---------- + self.sidebar = QVBoxLayout() + + self.btn_home = QPushButton("Home") + self.btn_start = QPushButton("Start Game") + self.btn_stop = QPushButton("Stop Game") + self.btn_setup = QPushButton("Setup Cameras") + self.btn_exit = QPushButton("Exit") + + for btn in [self.btn_home, self.btn_start, self.btn_stop, self.btn_setup, self.btn_exit]: + btn.setMinimumHeight(40) + self.sidebar.addWidget(btn) + + self.sidebar.addStretch(1) # push buttons to top + + # ---------- Content area (stack) ---------- + self.stack = QStackedWidget() + + # Home page + home_page = QWidget() + self.homeWidget = HomeWindow( + cameraList=self.cameraList, + on_cameraList_changed=self.on_cameraList_changed, + P_mats=self.P_mats, + logger=self.logger + ) + + home_layout = QVBoxLayout(home_page) + home_label = QLabel("Welcome to the Gaming Table Control") + home_label.setAlignment(Qt.AlignCenter) + + home_layout.addWidget(home_label) + home_layout.addWidget(self.homeWidget) + + # Setup page + self.setup_page = SetupPage( + cameraList=self.cameraList, + on_cameraList_changed=self.on_cameraList_changed, + logger=self.logger + ) + + # Options page (placeholder) + stop_page = QWidget() + stop_layout = QVBoxLayout(stop_page) + stop_label = QLabel("The game was stopped") + stop_label.setAlignment(Qt.AlignCenter) + stop_layout.addWidget(stop_label) + + # Add pages to stack + self.page_home_index = self.stack.addWidget(home_page) + self.page_setup_index = self.stack.addWidget(self.setup_page) + self.page_stop_index = self.stack.addWidget(stop_page) + + # ---------- Assemble layout ---------- + self.main_interaction_layout.addLayout(self.sidebar) # left side + self.main_interaction_layout.addWidget(self.stack, 1) # right side grows + + self.main_layout.addLayout(self.main_interaction_layout) + self.main_layout.addWidget(self.console) + + # ---------- Connections ---------- + self.btn_home.clicked.connect(self.show_home) + self.btn_setup.clicked.connect(self.show_setup) + self.btn_stop.clicked.connect(self.show_stop) + self.btn_start.clicked.connect(self.start_game) + self.btn_exit.clicked.connect(self.close) + + self.game_window = None + + # -------------------- + def on_cameraList_changed(self): + self.homeWidget.updateCameras() + self.logger.log("Camera List changed") + for camera in self.cameraList: + self.logger.log(str(camera.index)) + + + def show_home(self): + self.stack.setCurrentIndex(self.page_home_index) + + def show_setup(self): + self.stack.setCurrentIndex(self.page_setup_index) + + def show_stop(self): + self.stack.setCurrentIndex(self.page_stop_index) + + if(self.tracking) is not None: + self.tracking.stop() + + def start_game(self): + self.tracking = TrackingController( + cameraList=self.cameraList, + P_mats=self.P_mats, + logger=self.logger, + tick_ms=30 + ) + + self.tracking.start() + +def main(): + app = QApplication(sys.argv) + win = MainWindow() + win.show() + sys.exit(app.exec()) + + +if __name__ == "__main__": + main()