diff --git a/Table_edge.3mf b/Table_edge.3mf new file mode 100644 index 0000000..5f09dc9 Binary files /dev/null and b/Table_edge.3mf differ diff --git a/Table_edge.blend b/Table_edge.blend new file mode 100644 index 0000000..b8dbf54 Binary files /dev/null and b/Table_edge.blend differ diff --git a/Table_edge.blend1 b/Table_edge.blend1 new file mode 100644 index 0000000..3175099 Binary files /dev/null and b/Table_edge.blend1 differ diff --git a/Table_edge.stl b/Table_edge.stl new file mode 100644 index 0000000..9ccb954 Binary files /dev/null and b/Table_edge.stl differ diff --git a/Table_edge1.stl b/Table_edge1.stl new file mode 100644 index 0000000..c9fc1a9 Binary files /dev/null and b/Table_edge1.stl differ diff --git a/__pycache__/config.cpython-311.pyc b/__pycache__/config.cpython-311.pyc index adf4d91..68dba21 100644 Binary files a/__pycache__/config.cpython-311.pyc and b/__pycache__/config.cpython-311.pyc differ diff --git a/__pycache__/hand_detection.cpython-311.pyc b/__pycache__/hand_detection.cpython-311.pyc index 46c4da7..77ff2d1 100644 Binary files a/__pycache__/hand_detection.cpython-311.pyc and b/__pycache__/hand_detection.cpython-311.pyc differ diff --git a/__pycache__/helper.cpython-311.pyc b/__pycache__/helper.cpython-311.pyc index b5fb324..38344e5 100644 Binary files a/__pycache__/helper.cpython-311.pyc and b/__pycache__/helper.cpython-311.pyc differ diff --git a/__pycache__/hud.cpython-311.pyc b/__pycache__/hud.cpython-311.pyc index 1df254d..bac8df7 100644 Binary files a/__pycache__/hud.cpython-311.pyc and b/__pycache__/hud.cpython-311.pyc differ diff --git a/__pycache__/setup.cpython-311.pyc b/__pycache__/setup.cpython-311.pyc new file mode 100644 index 0000000..7006b1a Binary files /dev/null and b/__pycache__/setup.cpython-311.pyc differ diff --git a/config.py b/config.py index b3288d1..b9c83fa 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,7 @@ import numpy as np # ----- Configuration ----- -CAMERA_INDICES = [0] # List of camera device indices +CAMERA_INDICES_TO_CHECK = 10 FRAME_WIDTH = 3980 FRAME_HEIGHT = 2560 PIXELS_PER_INCH = 38 # will be set by calibration @@ -24,4 +24,7 @@ TAB_GAP = 10 LEFT_BTN_TOP = (TAB_X, TAB_Y0+2*(TAB_H+TAB_GAP)+20) RIGHT_BTN_TOP = (TAB_X, LEFT_BTN_TOP[1]+TAB_H+TAB_GAP) -DEBUG = True \ No newline at end of file +DEBUG = True + +TARGET_FPS = 40 +FRAME_INTERVAL = 1.0 / TARGET_FPS \ No newline at end of file diff --git a/hand_detection.py b/hand_detection.py index a443a3b..b3698b4 100644 --- a/hand_detection.py +++ b/hand_detection.py @@ -3,12 +3,18 @@ import cv2, numpy as np, time, fitz from helper import compute_distance from config import PINCH_THRESHOLD, RELEASE_THRESHOLD +# hand_state.py +from dataclasses import dataclass, field +from typing import Tuple, Dict, Optional + +from helper import finger_straight + # ----- Initialize Hand Detector ----- mp_hands = mp.solutions.hands mp_draw = mp.solutions.drawing_utils hands = mp_hands.Hands( static_image_mode=False, - max_num_hands=2, + max_num_hands=8, min_detection_confidence=0.7, min_tracking_confidence=0.5 ) @@ -16,10 +22,106 @@ hands = mp_hands.Hands( start_pt = None # measurement start point measuring = False # measurement in progress +@dataclass +class HandState: + id: int # a persistent identifier for this hand + landmarks: Dict[str, Tuple[int,int]] = field(default_factory=dict) + finger_straightness: Dict[str, float] = field(default_factory=dict) + gesture: Optional[str] = None # e.g. "pinch", "fist", "open" + gesture_persistence: int = 0 # how many frames the current gesture has held + +#handList = Dict[int, HandState] = {} +next_id = 0 + +def update(mp_results, frame_vis): + h, w, _ = frame_vis.shape + detected_centroids = [] + landmarks_list = [] + + # 1) pull out centroids & raw landmarks + if mp_results.multi_hand_landmarks: + for hand in mp_results.multi_hand_landmarks: + pts = [] + for lm in hand.landmark: + pts.append((int(lm.x*w), int(lm.y*h))) + centroid = np.mean(pts, axis=0) + detected_centroids.append(tuple(centroid.astype(int))) + landmarks_list.append((hand, pts)) + + # 2) match to existing by nearest centroid + new_hands = {} + used_ids = set() + for (hand, pts), centroid in zip(landmarks_list, detected_centroids): + # find best existing hand + best_id, best_dist = None, 1e9 + for hid, state in handList.items(): + dx, dy = np.array(state.landmarks['centroid']) - centroid + d = np.hypot(dx, dy) + if d < best_dist and d < 100: # 100px max match distance + best_dist, best_id = d, hid + + if best_id is None: + hid = next_id + next_id += 1 + state = HandState(id=hid) + else: + hid = best_id + state = handList[hid] + + used_ids.add(hid) + + # 3) update state + state.persistence += 1 + state.landmarks['centroid'] = centroid + + # compute fingertip positions + idx_tip = pts[mp.solutions.hands.HandLandmark.INDEX_FINGER_TIP] + mid_tip = pts[mp.solutions.hands.HandLandmark.MIDDLE_FINGER_TIP] + state.landmarks['index_tip'] = idx_tip + state.landmarks['middle_tip'] = mid_tip + + for name, tip_i, pip_i, mcp_i in [ + ('index', mp.solutions.hands.HandLandmark.INDEX_FINGER_TIP, + mp.solutions.hands.HandLandmark.INDEX_FINGER_PIP, + mp.solutions.hands.HandLandmark.INDEX_FINGER_MCP), + ('middle', mp.solutions.hands.HandLandmark.MIDDLE_FINGER_TIP, + mp.solutions.hands.HandLandmark.MIDDLE_FINGER_PIP, + mp.solutions.hands.HandLandmark.MIDDLE_FINGER_MCP), + # add ring, pinky, thumb similarly... + ]: + p_tip = pts[tip_i] + p_pip = pts[pip_i] + p_mcp = pts[mcp_i] + state.finger_straightness[name] = finger_straight(p_tip, p_pip, p_mcp) + + # 5) simple gesture detection + if state.finger_straightness['index'] < 0.2 and \ + state.finger_straightness['middle'] < 0.2: + gesture = 'fist' + elif state.finger_straightness['index'] > 0.8 and \ + state.finger_straightness['middle'] > 0.8: + gesture = 'open' + else: + gesture = None + + if gesture == state.gesture: + state.gesture_persistence += 1 + else: + state.gesture = gesture + state.gesture_persistence = 0 + + new_hands[hid] = state + + # 6) drop hands not seen this frame + handList = new_hands + return list(hands.values()) + + + def detect_hands(frame): rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) rgb.flags.writeable = False - results = hands.process(rgb) + results = hands.process(rgb) return results @@ -47,6 +149,8 @@ def check_hand(hand, frame_vis, object_calibrating): return fingertip_idx, measuring, start_pt + + def is_measuring(frame_vis, hand, object_calibrating): global measuring, start_pt h, w, _ = frame_vis.shape diff --git a/helper.py b/helper.py index e88ec5e..02dc0fa 100644 --- a/helper.py +++ b/helper.py @@ -1,6 +1,21 @@ import numpy as np +import cv2 # ----- Helper Functions ----- def compute_distance(p1, p2): return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) +# 4) compute straightness per finger +def finger_straight(i_tip, i_pip, i_mcp): + d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip))) + d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp))) + return float(np.clip(d1/(d2+1e-3), 0, 1)) + +def yes_no(): + while True: + key = cv2.waitKey(1) & 0xFF + + if key == ord("y"): + return True + if key == ord("n"): + return False diff --git a/hud.py b/hud.py index 724593e..5fc6cce 100644 --- a/hud.py +++ b/hud.py @@ -45,81 +45,6 @@ def find_hud_placement(frame_vis): return arrow_tip, dir_vec -# —————————————————————————————————————————————— -# PDF loading / rasterization at import time -# —————————————————————————————————————————————— -PDF_PATH = "DarkAngles.pdf" # change this to your file -PDF_DPI = 600 # controls resolution of rasterization - -_doc = fitz.open(PDF_PATH) -_pdf_pages = [] -for page in _doc: - # render page to pixmap at desired zoom - zoom = PDF_DPI / 72.0 - mat = fitz.Matrix(zoom, zoom) - pix = page.get_pixmap(matrix=mat, alpha=False) - # convert pixmap to ndarray - img = np.frombuffer(pix.samples, dtype=np.uint8) - img = img.reshape(pix.height, pix.width, pix.n) - if pix.n == 4: - img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR) - _pdf_pages.append(img) -_doc.close() - -# —————————————————————————————————————————————— -# HUD drawing routines -# —————————————————————————————————————————————— - -# rotated rectangle → 4 pts -def _rect_to_pts(center, size, angle_rad): - cx, cy = center - w, h = size - # local corners - pts = np.array([ - [-w/2, -h/2], - [ w/2, -h/2], - [ w/2, h/2], - [-w/2, h/2], - ]) - # rotation - c, s = np.cos(angle_rad), np.sin(angle_rad) - R = np.array([[c, -s],[s, c]]) - pts = pts.dot(R.T) - pts += np.array([cx, cy]) - return pts.astype(np.float32) - -def draw_hud_box(frame, tip, angle_rad, size=(120, 60), color=(255,0,255), thickness=2): - """Draw a rotated HUD rectangle at `tip` pointing along `angle_rad`.""" - # compute box corners - pts = _rect_to_pts(center=tip, size=size, angle_rad=angle_rad) - cv2.drawContours(frame, [pts.astype(int)], -1, color, thickness) - # label - cv2.putText(frame, "HUD", (int(tip[0]+5), int(tip[1]+5)), cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) - return pts # return the quad for PDF warping - -def draw_pdf_page(frame, page_index, dst_quad): - """ - Warp PDF page image #page_index into the quadrilateral dst_quad. - dst_quad: 4×2 float32 array of destination corners in clock-wise order. - """ - if page_index < 0 or page_index >= len(_pdf_pages): - return - src = _pdf_pages[page_index] - h, w = src.shape[:2] - - # source corners (tl, tr, br, bl) - src_quad = np.array([[0,0], [w,0], [w,h], [0,h]], dtype=np.float32) - # compute homography - M = cv2.getPerspectiveTransform(src_quad, dst_quad) - # warp PDF page into place (transparent where outside) - warp = cv2.warpPerspective(src, M, (frame.shape[1], frame.shape[0])) - mask = cv2.warpPerspective(np.ones((h,w), dtype=np.uint8)*255, M, (frame.shape[1], frame.shape[0])) - # composite onto frame - inv = cv2.bitwise_not(mask) - bg = cv2.bitwise_and(frame, frame, mask=inv) - fg = cv2.bitwise_and(warp, warp, mask=mask) - np.copyto(frame, bg+fg) - # hud.py class HUD: def __init__(self, marker_id, pdf_pages, default_page=0): diff --git a/main.py b/main.py index 22843e5..1ab5fa3 100644 --- a/main.py +++ b/main.py @@ -5,10 +5,12 @@ from helper import compute_distance from hud import find_hud_placement import math -from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import FRAME_INTERVAL, TARGET_FPS from config import DEBUG - -from hud import draw_hud_box, draw_pdf_page +from setup import InitCameras +#from hud import draw_hud_box, draw_pdf_page +import time # ----- Global State ----- measuring = False # measurement in progress @@ -22,7 +24,7 @@ hud_rot = None register_hud = False # ----- Per-camera Processing ----- -def process_frame(frame): +def process_frame(frame, camera_id: int = 0): global measuring, start_pt, calibrating global object_calibrating global fingertip_idx_global @@ -53,8 +55,6 @@ def process_frame(frame): else: res = hud_pos, hud_rot - - # create proj output projection_out = np.zeros_like(frame) @@ -76,8 +76,8 @@ def process_frame(frame): dx, dy = dir_vec arrow_angle = math.atan2(dy, dx) # result in radians - quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400)) - draw_pdf_page(projection_out, page_index=0, dst_quad=quad) + #quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400)) + #draw_pdf_page(projection_out, page_index=0, dst_quad=quad) if register_hud: hud_pos = arrow_tip @@ -92,31 +92,30 @@ def process_frame(frame): # ----- Main ----- ------------------------------------------------------------------------------------- def main(): - all_captures = [] - for idx in CAMERA_INDICES: - capture = cv2.VideoCapture(idx) - capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) - capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) - all_captures.append(capture) - if not all(capture.isOpened() for capture in all_captures): - print("Error: could not open all cameras") - return - + all_captures = InitCameras() + + print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)") print("Press 'o' for circle calib, 'q' to quit.") - while True: - frames = [capture.read()[1] for capture in all_captures] - frame = next((f for f in frames if f is not None), None) - if frame is None: + last_time = time.time() + while True: + processed_frames = [] + + for camera_id, capture in enumerate(all_captures): + ok, frame = capture.read() + if not ok or frame is None: + continue + + projection, debug = process_frame(frame, camera_id) + processed_frames.append((camera_id, projection, debug)) + + if not processed_frames: break - # process - projection, debug = process_frame(frame) - - # show windows - if DEBUG: - cv2.imshow('Debug Output', debug) - cv2.imshow('Projector Output', projection) + for camera_id, projection, debug in processed_frames: + if DEBUG: + cv2.imshow(f'Debug Output (cam {camera_id})', debug) + cv2.imshow(f'Projector Output (cam {camera_id})', projection) key = cv2.waitKey(1) & 0xFF @@ -131,10 +130,18 @@ def main(): register_hud = True hud_pos = None + # ——— frame‐rate limiting ——— + now = time.time() + elapsed = now - last_time + to_wait = FRAME_INTERVAL - elapsed + if to_wait > 0: + time.sleep(to_wait) + last_time = time.time() + for capture in all_captures: capture.release() cv2.destroyAllWindows() if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e5d74e6 --- /dev/null +++ b/setup.py @@ -0,0 +1,81 @@ +import cv2 +import numpy as np + +from config import CAMERA_INDICES_TO_CHECK, FRAME_WIDTH, FRAME_HEIGHT + +def _preview_camera(capture: cv2.VideoCapture, cameraID: int) -> str: + """Show a live preview and return the user's choice: 'add', 'skip', or 'quit'.""" + window_name = f"Camera {cameraID}" + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + print(f"[Camera {cameraID}] Press 'y' to add, 'n' to skip, 'q' to stop scanning.") + + try: + while True: + ok, frame = capture.read() + if not ok: + print(f"[Camera {cameraID}] Failed to read frame; skipping.") + return "skip" + + cv2.imshow(window_name, frame) + + key = cv2.waitKey(1) & 0xFF + + if key == ord("y"): + print(f"[Camera {cameraID}] Selected.") + return "add" + if key == ord("n"): + print(f"[Camera {cameraID}] Skipped.") + return "skip" + if key == ord("q") or key == 27: # 27 == ESC + print(f"[Camera {cameraID}] Stopping camera scan.") + return "quit" + finally: + cv2.destroyWindow(window_name) + + +def InitCameras(): + allCaptures = [] + + setup_window = "Camera Setup" + instructions = np.zeros((240, 560, 3), dtype=np.uint8) + cv2.putText(instructions, "Camera Setup", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 200, 255), 2) + cv2.putText(instructions, "Each camera will preview in its own window.", (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) + cv2.putText(instructions, "Use 'y' to add, 'n' to skip, 'q'/ESC to finish.", (20, 125), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) + cv2.putText(instructions, "Close this window when you're done.", (20, 160), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1) + + cv2.namedWindow(setup_window, cv2.WINDOW_AUTOSIZE) + cv2.imshow(setup_window, instructions) + cv2.waitKey(1) + + try: + for cameraID in range(CAMERA_INDICES_TO_CHECK): + capture = cv2.VideoCapture(cameraID) + + if not capture.isOpened(): + capture.release() + continue + + capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) + capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) + + decision = _preview_camera(capture, cameraID) + + if decision == "add": + allCaptures.append(capture) + else: + capture.release() + + if decision == "quit": + break + + if not all(capture.isOpened() for capture in allCaptures): + print("Error: could not open all selected cameras.") + return + + if not allCaptures: + print("No cameras selected.") + return + + return allCaptures + finally: + cv2.destroyWindow(setup_window)