diff --git a/DarkAngles.pdf b/DarkAngles.pdf new file mode 100644 index 0000000..c489527 Binary files /dev/null and b/DarkAngles.pdf differ diff --git a/Stuff/Untitled-1.txt b/Stuff/Untitled-1.txt index 57816ac..7227d5a 100644 --- a/Stuff/Untitled-1.txt +++ b/Stuff/Untitled-1.txt @@ -1,195 +1,150 @@ -import cv2 -import mediapipe as mp -import numpy as np +import cv2, numpy as np, time, fitz +from calibration import object_calibration, draw_cal_circle +from hand_detection import detect_hands, check_hand +from helper import compute_distance +from hud import find_hud_placement, draw_hud +import math -# ----- Helper Functions ----- -def compute_distance(p1, p2): - return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) +from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import DEBUG -# ----- Configuration ----- -CAMERA_INDICES = [0] # List of camera device indices -FRAME_WIDTH = 1280 -FRAME_HEIGHT = 720 -PIXELS_PER_INCH = 20 # will be set by calibration -PINCH_THRESHOLD = 40 # px to start touch -RELEASE_THRESHOLD = 60 # px to end touch -CIRCLE_TOUCH_THRESHOLD = 20 # px tolerance for circle touch +from hud import draw_hud_box, draw_pdf_page -# ----- Initialize Hand Detector ----- -mp_hands = mp.solutions.hands -mp_draw = mp.solutions.drawing_utils -hands = mp_hands.Hands( - static_image_mode=False, - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 -) # ----- Global State ----- measuring = False # measurement in progress start_pt = None # measurement start point calibrating = False # pinch-based calibration flag -cal_start = None # pinch calibration start point object_calibrating = False # object calibration flag -cal_circle = None # reference circle (x,y,r) fingertip_idx_global = None # last detected fingertip position + +hud_pos = None +hud_rot = None +register_hud = False + # ----- Per-camera Processing ----- def process_frame(frame): - global measuring, start_pt, calibrating, cal_start, object_calibrating, cal_circle, PIXELS_PER_INCH, fingertip_idx_global + global measuring, start_pt, calibrating + global object_calibrating + global fingertip_idx_global + + global register_hud, hud_pos, hud_rot + frame_out = cv2.flip(frame, 1) - h, w, _ = frame_out.shape + + frame_vis = frame_out.copy() + projection_debug = frame_out.copy() # Hand detection - rgb = cv2.cvtColor(frame_out, cv2.COLOR_BGR2RGB) - rgb.flags.writeable = False - results = hands.process(rgb) - rgb.flags.writeable = True - frame_vis = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) + + results = detect_hands(frame_out) fingertip_idx = None - fingertip_mid = None - if results.multi_hand_landmarks: hand = results.multi_hand_landmarks[0] - mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS) - # get index and middle finger tips and PIP joints - idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] - idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP] - mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP] - mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP] - ix, iy = int(idx_tip.x * w), int(idx_tip.y * h) - mx, my = int(mid_tip.x * w), int(mid_tip.y * h) - fingertip_idx = (ix, iy) - fingertip_mid = (mx, my) - fingertip_idx_global = fingertip_idx - # draw fingertips - cv2.circle(frame_vis, fingertip_idx, 8, (0,255,0), -1) - cv2.circle(frame_vis, fingertip_mid, 8, (0,255,0), -1) - # check extension - index_ext = idx_tip.y < idx_pip.y - middle_ext = mid_tip.y < mid_pip.y - # if measurement in progress but fingers no longer both extended, stop measuring - if measuring and not (index_ext and middle_ext): - measuring = False - # pinch distance - pinch = compute_distance(fingertip_idx, fingertip_mid) - cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2) - - # pinch calibration - if calibrating: - if pinch < PINCH_THRESHOLD and cal_start is None: - cal_start = fingertip_idx - print("Pinch calibration start set") - elif pinch > RELEASE_THRESHOLD and cal_start is not None: - cal_end = fingertip_idx - px = compute_distance(cal_start, cal_end) - inches = float(input("Enter actual distance between points (inches): ")) - PIXELS_PER_INCH = px / inches - print(f"Calibrated: {PIXELS_PER_INCH:.2f} px/inch") - calibrating = False - cal_start = None - # measurement gesture - elif not object_calibrating and index_ext and middle_ext: - if pinch < PINCH_THRESHOLD and not measuring: - measuring = True - start_pt = fingertip_idx - elif pinch > RELEASE_THRESHOLD and measuring: - measuring = False + fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating) + fingertip_idx_global = fingertip_idx + # object calibration if object_calibrating and fingertip_idx is not None: - hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255])) - masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask) - gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY) - gray = cv2.medianBlur(gray,5) - circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100, - param1=50, param2=30, minRadius=10, maxRadius=300) - if circles is not None: - circles = np.round(circles[0]).astype(int) - touched = [(x,y,r) for x,y,r in circles - if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD] - if touched: - touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2])) - x,y,r = touched[0] - cal_circle = (x,y,r) - PIXELS_PER_INCH = 2 * r - print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch") - object_calibrating = False + object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD) + object_calibrating = False + - # permanent reference circle - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2) - cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1) - cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1) + res = None + if hud_pos is None: + res = find_hud_placement(frame_vis) + else: + res = hud_pos, hud_rot + + if res is not None: + arrow_tip, dir_vec = res + draw_hud(frame_vis, arrow_tip, dir_vec) + + dx, dy = dir_vec + arrow_angle = math.atan2(dy, dx) # result in radians + quad = draw_hud_box(frame_vis, arrow_tip, arrow_angle, size=(400,400)) + draw_pdf_page(frame_vis, page_index=0, dst_quad=quad) + + if register_hud: + hud_pos = arrow_tip + hud_rot = dir_vec + register_hud = False return frame_vis -# ----- Main ----- +# ----- Main ----- ------------------------------------------------------------------------------------- def main(): - caps = [] + global register_hud, hud_pos, hud_rot + + all_captures = [] for idx in CAMERA_INDICES: - cap = cv2.VideoCapture(idx) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) - caps.append(cap) - if not all(cap.isOpened() for cap in caps): + capture = cv2.VideoCapture(idx) + capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) + capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) + all_captures.append(capture) + if not all(capture.isOpened() for capture in all_captures): print("Error: could not open all cameras") return - print("Press 'c' for pinch calib, 'o' for circle calib, 'q' to quit.") + print("Press 'o' for circle calib, 'q' to quit.") while True: - frames = [cap.read()[1] for cap in caps] + frames = [capture.read()[1] for capture in all_captures] frame = next((f for f in frames if f is not None), None) + if frame is None: break + # process full_view = process_frame(frame) + # create proj output - proj = np.zeros_like(full_view) - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(proj,(cx,cy),cr,(0,0,255),2) + projection_out = np.zeros_like(full_view) + if measuring and start_pt and fingertip_idx_global: - cv2.line(proj, start_pt, fingertip_idx_global, (255,0,0),2) + cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2) px = compute_distance(start_pt, fingertip_idx_global) inch = px/PIXELS_PER_INCH; cm = inch*2.54 mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) - cv2.putText(proj,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) + cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), + cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2) + + if hud_pos: + dx, dy = hud_rot + arrow_angle = math.atan2(dy, dx) # result in radians + quad = draw_hud_box(projection_out, hud_pos, arrow_angle, size=(400,400)) + draw_pdf_page(projection_out, page_index=0, dst_quad=quad) + # overlay proj onto debug debug = full_view.copy() - # overlay ref circle - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(debug,(cx,cy),cr,(0,0,255),2) + # overlay measurement if measuring and start_pt and fingertip_idx_global: - cv2.line(debug, start_pt, fingertip_idx_global, (255,0,0),2) + cv2.line(debug, start_pt, fingertip_idx_global, (255,255,0),2) cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) + cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) # show windows - cv2.imshow('Hand Measure', debug) - cv2.imshow('Projector Output', proj) + cv2.imshow('Debug Output', debug) + cv2.imshow('Projector Output', projection_out) key = cv2.waitKey(1) & 0xFF + if key == ord('q'): break - elif key == ord('c'): - global calibrating - calibrating = True - cal_start = None - print("Entered pinch calibration mode.") elif key == ord('o'): global object_calibrating object_calibrating = True print("Entered circle calibration mode.") + elif key == ord('j'): + register_hud = True + hud_pos = None + + for capture in all_captures: + capture.release() - for cap in caps: - cap.release() cv2.destroyAllWindows() if __name__ == '__main__': diff --git a/yolo11n.pt b/Stuff/yolo11n.pt similarity index 100% rename from yolo11n.pt rename to Stuff/yolo11n.pt diff --git a/Table.blend b/Table.blend index a73139e..500f4f1 100644 Binary files a/Table.blend and b/Table.blend differ diff --git a/Table.blend1 b/Table.blend1 new file mode 100644 index 0000000..4e36a05 Binary files /dev/null and b/Table.blend1 differ diff --git a/Table1.blend b/Table1.blend new file mode 100644 index 0000000..8ee8284 Binary files /dev/null and b/Table1.blend differ diff --git a/Table1.blend1 b/Table1.blend1 new file mode 100644 index 0000000..753caad Binary files /dev/null and b/Table1.blend1 differ diff --git a/TableIdk.py b/TableIdk.py deleted file mode 100644 index 43ff530..0000000 --- a/TableIdk.py +++ /dev/null @@ -1,279 +0,0 @@ -import cv2 -import mediapipe as mp -import numpy as np - -# ----- Helper Functions ----- -def compute_distance(p1, p2): - return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) - -# ----- Configuration ----- -CAMERA_INDICES = [0] # List of camera device indices -FRAME_WIDTH = 1280 -FRAME_HEIGHT = 720 -PIXELS_PER_INCH = 38 # will be set by calibration -PINCH_THRESHOLD = 40 # px to start touch -RELEASE_THRESHOLD = 60 # px to end touch -CIRCLE_TOUCH_THRESHOLD = 20 # px tolerance for circle touch - -# HSV range for shape color (tune for your arrow: now tailored for orange) -LOWER_SHAPE = np.array([10, 100, 100]) # hue from 10° (orange) to -UPPER_SHAPE = np.array([30, 255, 255]) # hue up to 30°, full sat/val range - -# dynamic list of registered arrows -# each entry will be {'lower': np.array, 'upper': np.array, 'tip': (x,y) or None} -shape_ranges = [] - -# ----- Initialize Hand Detector ----- -mp_hands = mp.solutions.hands -mp_draw = mp.solutions.drawing_utils -hands = mp_hands.Hands( - static_image_mode=False, - max_num_hands=1, - min_detection_confidence=0.7, - min_tracking_confidence=0.5 -) - -# ----- Global State ----- -measuring = False # measurement in progress -start_pt = None # measurement start point -calibrating = False # pinch-based calibration flag -cal_start = None # pinch calibration start point -object_calibrating = False # object calibration flag -cal_circle = None # reference circle (x,y,r) -fingertip_idx_global = None # last detected fingertip position -arrow_tip = None # detected arrow tip position - -# ----- Per-camera Processing ----- -def process_frame(frame): - global measuring, start_pt, calibrating, cal_start - global object_calibrating, cal_circle, PIXELS_PER_INCH - global fingertip_idx_global, arrow_tip - - frame_out = cv2.flip(frame, 1) - h, w, _ = frame_out.shape - - # Hand detection - rgb = cv2.cvtColor(frame_out, cv2.COLOR_BGR2RGB) - rgb.flags.writeable = False - results = hands.process(rgb) - rgb.flags.writeable = True - frame_vis = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) - - fingertip_idx = None - fingertip_mid = None - - if results.multi_hand_landmarks: - hand = results.multi_hand_landmarks[0] - mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS) - # get index and middle finger tips and PIP joints - idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] - idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP] - mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP] - mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP] - ix, iy = int(idx_tip.x * w), int(idx_tip.y * h) - mx, my = int(mid_tip.x * w), int(mid_tip.y * h) - fingertip_idx = (ix, iy) - fingertip_mid = (mx, my) - fingertip_idx_global = fingertip_idx - # draw fingertips - cv2.circle(frame_vis, fingertip_idx, 8, (0,255,0), -1) - cv2.circle(frame_vis, fingertip_mid, 8, (0,255,0), -1) - # check extension - index_ext = idx_tip.y < idx_pip.y - middle_ext = mid_tip.y < mid_pip.y - # if measurement in progress but fingers no longer both extended, stop measuring - if measuring and not (index_ext and middle_ext): - measuring = False - # pinch distance - pinch = compute_distance(fingertip_idx, fingertip_mid) - cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2) - - # pinch calibration - if calibrating: - if pinch < PINCH_THRESHOLD and cal_start is None: - cal_start = fingertip_idx - print("Pinch calibration start set") - elif pinch > RELEASE_THRESHOLD and cal_start is not None: - cal_end = fingertip_idx - px = compute_distance(cal_start, cal_end) - inches = float(input("Enter actual distance between points (inches): ")) - PIXELS_PER_INCH = px / inches - print(f"Calibrated: {PIXELS_PER_INCH:.2f} px/inch") - calibrating = False - cal_start = None - # measurement gesture - elif not object_calibrating and index_ext and middle_ext: - if pinch < PINCH_THRESHOLD and not measuring: - measuring = True - start_pt = fingertip_idx - elif pinch > RELEASE_THRESHOLD and measuring: - measuring = False - - - - # object calibration - if object_calibrating and fingertip_idx is not None: - hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255])) - masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask) - gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY) - gray = cv2.medianBlur(gray,5) - circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100, - param1=50, param2=30, minRadius=10, maxRadius=300) - if circles is not None: - circles = np.round(circles[0]).astype(int) - touched = [(x,y,r) for x,y,r in circles - if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD] - if touched: - touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2])) - x,y,r = touched[0] - cal_circle = (x,y,r) - PIXELS_PER_INCH = 2 * r - print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch") - object_calibrating = False - - # permanent reference circle - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2) - cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1) - cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1) - - # ----- Arrow Shape Detection & HUD Placement ----- - # 1) Build a clean orange mask - hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) - mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE) - kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5)) - mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern) - mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern) - - # 2) Find and filter contours - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - for cnt in contours: - area = cv2.contourArea(cnt) - if area < 1000: - continue - - # Approximate to polygon and require exactly 5 corners - peri = cv2.arcLength(cnt, True) - approx = cv2.approxPolyDP(cnt, 0.02 * peri, True) - if len(approx) != 5: - continue - - pts = approx.reshape(-1,2) - centroid = np.mean(pts, axis=0) - - # Find the arrow tip as the corner farthest from centroid - dists = [np.linalg.norm(pt - centroid) for pt in pts] - tip_pt = pts[int(np.argmax(dists))] - raw_tip = (int(tip_pt[0]), int(tip_pt[1])) - - # 3) Smooth the tip over time - alpha = 0.2 - if arrow_tip is None: - arrow_tip = raw_tip - else: - arrow_tip = ( - int(alpha * raw_tip[0] + (1-alpha) * arrow_tip[0]), - int(alpha * raw_tip[1] + (1-alpha) * arrow_tip[1]) - ) - - # Draw the arrow & tip - cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2) - cv2.circle(frame_vis, arrow_tip, 8, (0,255,255), -1) - cv2.putText(frame_vis, "Arrow Tip", arrow_tip, - cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,255), 2) - - # Place the HUD box in the arrow’s pointing direction - dir_vec = tip_pt - centroid - norm = np.linalg.norm(dir_vec) - if norm>0: - dir_unit = dir_vec / norm - offset = 50 - hud_center = (int(arrow_tip[0] + dir_unit[0]*offset), - int(arrow_tip[1] + dir_unit[1]*offset)) - - # Build and rotate a 120×60 HUD rectangle - w2, h2 = 60, 30 - theta = np.arctan2(dir_unit[1], dir_unit[0]) - R = np.array([[ np.cos(theta), -np.sin(theta)], - [ np.sin(theta), np.cos(theta)]]) - corners = np.array([[-w2,-h2], [w2,-h2], [w2,h2], [-w2,h2]]) - hud_pts = (corners @ R.T) + np.array(hud_center) - hud_pts = hud_pts.astype(int) - - cv2.drawContours(frame_vis, [hud_pts], -1, (255,0,255), 2) - cv2.putText(frame_vis, "HUD", hud_center, - cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255,0,255), 2) - - - - return frame_vis - -# ----- Main ----- -def main(): - caps = [] - for idx in CAMERA_INDICES: - cap = cv2.VideoCapture(idx) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) - caps.append(cap) - if not all(cap.isOpened() for cap in caps): - print("Error: could not open all cameras") - return - - print("Press 'c' for pinch calib, 'o' for circle calib, 'q' to quit.") - while True: - frames = [cap.read()[1] for cap in caps] - frame = next((f for f in frames if f is not None), None) - if frame is None: - break - # process - full_view = process_frame(frame) - # create proj output - proj = np.zeros_like(full_view) - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(proj,(cx,cy),cr,(0,0,255),2) - if measuring and start_pt and fingertip_idx_global: - cv2.line(proj, start_pt, fingertip_idx_global, (255,0,0),2) - px = compute_distance(start_pt, fingertip_idx_global) - inch = px/PIXELS_PER_INCH; cm = inch*2.54 - mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) - cv2.putText(proj,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) - # overlay proj onto debug - debug = full_view.copy() - # overlay ref circle - if cal_circle: - cx,cy,cr = cal_circle - cv2.circle(debug,(cx,cy),cr,(0,0,255),2) - # overlay measurement - if measuring and start_pt and fingertip_idx_global: - cv2.line(debug, start_pt, fingertip_idx_global, (255,0,0),2) - cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), - cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2) - - # show windows - cv2.imshow('Hand Measure', debug) - cv2.imshow('Projector Output', proj) - - key = cv2.waitKey(1) & 0xFF - if key == ord('q'): - break - elif key == ord('c'): - global calibrating - calibrating = True - cal_start = None - print("Entered pinch calibration mode.") - elif key == ord('o'): - global object_calibrating - object_calibrating = True - print("Entered circle calibration mode.") - - for cap in caps: - cap.release() - cv2.destroyAllWindows() - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__pycache__/calibration.cpython-311.pyc b/__pycache__/calibration.cpython-311.pyc new file mode 100644 index 0000000..ce9a06f Binary files /dev/null and b/__pycache__/calibration.cpython-311.pyc differ diff --git a/__pycache__/config.cpython-311.pyc b/__pycache__/config.cpython-311.pyc new file mode 100644 index 0000000..adf4d91 Binary files /dev/null and b/__pycache__/config.cpython-311.pyc differ diff --git a/__pycache__/hand_detection.cpython-311.pyc b/__pycache__/hand_detection.cpython-311.pyc new file mode 100644 index 0000000..46c4da7 Binary files /dev/null and b/__pycache__/hand_detection.cpython-311.pyc differ diff --git a/__pycache__/helper.cpython-311.pyc b/__pycache__/helper.cpython-311.pyc new file mode 100644 index 0000000..b5fb324 Binary files /dev/null and b/__pycache__/helper.cpython-311.pyc differ diff --git a/__pycache__/hud.cpython-311.pyc b/__pycache__/hud.cpython-311.pyc new file mode 100644 index 0000000..1df254d Binary files /dev/null and b/__pycache__/hud.cpython-311.pyc differ diff --git a/calibration.py b/calibration.py new file mode 100644 index 0000000..df12adf --- /dev/null +++ b/calibration.py @@ -0,0 +1,32 @@ +import cv2, numpy as np + +from helper import compute_distance + +cal_circle = None # reference circle (x,y,r) + +def object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD): + global cal_circle + hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) + mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255])) + masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask) + gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY) + gray = cv2.medianBlur(gray,5) + circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100, + param1=50, param2=30, minRadius=10, maxRadius=300) + if circles is not None: + circles = np.round(circles[0]).astype(int) + touched = [(x,y,r) for x,y,r in circles + if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD] + if touched: + touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2])) + x,y,r = touched[0] + cal_circle = (x,y,r) + PIXELS_PER_INCH = 2 * r + print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch") + +def draw_cal_circle(frame_vis): + global cal_circle + cx,cy,cr = cal_circle + cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2) + cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1) + cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1) \ No newline at end of file diff --git a/camera.py b/camera.py new file mode 100644 index 0000000..e69de29 diff --git a/config.py b/config.py new file mode 100644 index 0000000..b3288d1 --- /dev/null +++ b/config.py @@ -0,0 +1,27 @@ +import numpy as np + +# ----- Configuration ----- +CAMERA_INDICES = [0] # List of camera device indices +FRAME_WIDTH = 3980 +FRAME_HEIGHT = 2560 +PIXELS_PER_INCH = 38 # will be set by calibration +PINCH_THRESHOLD = 40 # px to start touch +RELEASE_THRESHOLD = 60 # px to end touch +CIRCLE_TOUCH_THRESHOLD = 20 # px tolerance for circle touch + +# HSV range for shape color (tune for your arrow: now tailored for orange) +LOWER_SHAPE = np.array([10, 100, 100]) # hue from 10° (orange) to +UPPER_SHAPE = np.array([30, 255, 255]) # hue up to 30°, full sat/val range + +DEBOUNCE_TIME = 1.0 # seconds + +# ——— Constants for Tabs & Buttons ——— +TAB_W, TAB_H = 40, 60 +TAB_X = 600 # adjust this to your projector output width +TAB_Y0 = 50 +TAB_GAP = 10 +# left/right arrow buttons +LEFT_BTN_TOP = (TAB_X, TAB_Y0+2*(TAB_H+TAB_GAP)+20) +RIGHT_BTN_TOP = (TAB_X, LEFT_BTN_TOP[1]+TAB_H+TAB_GAP) + +DEBUG = True \ No newline at end of file diff --git a/hand_detection.py b/hand_detection.py new file mode 100644 index 0000000..a443a3b --- /dev/null +++ b/hand_detection.py @@ -0,0 +1,84 @@ +import mediapipe as mp +import cv2, numpy as np, time, fitz +from helper import compute_distance +from config import PINCH_THRESHOLD, RELEASE_THRESHOLD + +# ----- Initialize Hand Detector ----- +mp_hands = mp.solutions.hands +mp_draw = mp.solutions.drawing_utils +hands = mp_hands.Hands( + static_image_mode=False, + max_num_hands=2, + min_detection_confidence=0.7, + min_tracking_confidence=0.5 +) + +start_pt = None # measurement start point +measuring = False # measurement in progress + +def detect_hands(frame): + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + rgb.flags.writeable = False + results = hands.process(rgb) + + return results + + +def check_hand(hand, frame_vis, object_calibrating): + h, w, _ = frame_vis.shape + + fingertip_idx = None + fingertip_mid = None + + mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS) + # get index and middle finger tips and PIP joints + idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] + mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP] + ix, iy = int(idx_tip.x * w), int(idx_tip.y * h) + mx, my = int(mid_tip.x * w), int(mid_tip.y * h) + fingertip_idx = (ix, iy) + fingertip_mid = (mx, my) + + # draw fingertips + cv2.circle(frame_vis, fingertip_idx, 8, (255,255,0), -1) + cv2.circle(frame_vis, fingertip_mid, 8, (0,255,0), -1) + + measuring, start_pt = is_measuring(frame_vis, hand, object_calibrating) + + return fingertip_idx, measuring, start_pt + +def is_measuring(frame_vis, hand, object_calibrating): + global measuring, start_pt + h, w, _ = frame_vis.shape + + idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP] + idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP] + mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP] + mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP] + + ix, iy = int(idx_tip.x * w), int(idx_tip.y * h) + mx, my = int(mid_tip.x * w), int(mid_tip.y * h) + fingertip_idx = (ix, iy) + fingertip_mid = (mx, my) + + # check extension + index_ext = idx_tip.y < idx_pip.y + middle_ext = mid_tip.y < mid_pip.y + + # if measurement in progress but fingers no longer both extended, stop measuring + if measuring and not (index_ext and middle_ext): + measuring = False + + # pinch distance + pinch = compute_distance(fingertip_idx, fingertip_mid) + cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2) + + # measurement gesture + if not object_calibrating and index_ext and middle_ext: + if pinch < PINCH_THRESHOLD and not measuring: + measuring = True + start_pt = fingertip_idx + elif pinch > RELEASE_THRESHOLD and measuring: + measuring = False + + return measuring, start_pt \ No newline at end of file diff --git a/helper.py b/helper.py new file mode 100644 index 0000000..e88ec5e --- /dev/null +++ b/helper.py @@ -0,0 +1,6 @@ +import numpy as np + +# ----- Helper Functions ----- +def compute_distance(p1, p2): + return np.hypot(p2[0] - p1[0], p2[1] - p1[1]) + diff --git a/hud.py b/hud.py new file mode 100644 index 0000000..724593e --- /dev/null +++ b/hud.py @@ -0,0 +1,136 @@ +import cv2, numpy as np +from config import LOWER_SHAPE, UPPER_SHAPE +import cv2 +import numpy as np +import time +import fitz # PyMuPDF for PDF rendering + + + +def find_hud_placement(frame_vis): + # ----- Arrow Shape Detection & HUD Placement ----- + # 1) Build a clean orange mask + hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV) + mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE) + kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5)) + mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern) + + # 2) Find and filter contours + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + for cnt in contours: + area = cv2.contourArea(cnt) + if area < 1000: + continue + + # Approximate to polygon and require exactly 5 corners + peri = cv2.arcLength(cnt, True) + approx = cv2.approxPolyDP(cnt, 0.02 * peri, True) + if len(approx) != 5: + continue + + pts = approx.reshape(-1,2) + centroid = np.mean(pts, axis=0) + + # Find the arrow tip as the corner farthest from centroid + dists = [np.linalg.norm(pt - centroid) for pt in pts] + tip_pt = pts[int(np.argmax(dists))] + arrow_tip = (int(tip_pt[0]), int(tip_pt[1])) + + # Place the HUD box in the arrow’s pointing direction + dir_vec = tip_pt - centroid + + #ToDo remove + cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2) + + return arrow_tip, dir_vec + +# —————————————————————————————————————————————— +# PDF loading / rasterization at import time +# —————————————————————————————————————————————— +PDF_PATH = "DarkAngles.pdf" # change this to your file +PDF_DPI = 600 # controls resolution of rasterization + +_doc = fitz.open(PDF_PATH) +_pdf_pages = [] +for page in _doc: + # render page to pixmap at desired zoom + zoom = PDF_DPI / 72.0 + mat = fitz.Matrix(zoom, zoom) + pix = page.get_pixmap(matrix=mat, alpha=False) + # convert pixmap to ndarray + img = np.frombuffer(pix.samples, dtype=np.uint8) + img = img.reshape(pix.height, pix.width, pix.n) + if pix.n == 4: + img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR) + _pdf_pages.append(img) +_doc.close() + +# —————————————————————————————————————————————— +# HUD drawing routines +# —————————————————————————————————————————————— + +# rotated rectangle → 4 pts +def _rect_to_pts(center, size, angle_rad): + cx, cy = center + w, h = size + # local corners + pts = np.array([ + [-w/2, -h/2], + [ w/2, -h/2], + [ w/2, h/2], + [-w/2, h/2], + ]) + # rotation + c, s = np.cos(angle_rad), np.sin(angle_rad) + R = np.array([[c, -s],[s, c]]) + pts = pts.dot(R.T) + pts += np.array([cx, cy]) + return pts.astype(np.float32) + +def draw_hud_box(frame, tip, angle_rad, size=(120, 60), color=(255,0,255), thickness=2): + """Draw a rotated HUD rectangle at `tip` pointing along `angle_rad`.""" + # compute box corners + pts = _rect_to_pts(center=tip, size=size, angle_rad=angle_rad) + cv2.drawContours(frame, [pts.astype(int)], -1, color, thickness) + # label + cv2.putText(frame, "HUD", (int(tip[0]+5), int(tip[1]+5)), cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2) + return pts # return the quad for PDF warping + +def draw_pdf_page(frame, page_index, dst_quad): + """ + Warp PDF page image #page_index into the quadrilateral dst_quad. + dst_quad: 4×2 float32 array of destination corners in clock-wise order. + """ + if page_index < 0 or page_index >= len(_pdf_pages): + return + src = _pdf_pages[page_index] + h, w = src.shape[:2] + + # source corners (tl, tr, br, bl) + src_quad = np.array([[0,0], [w,0], [w,h], [0,h]], dtype=np.float32) + # compute homography + M = cv2.getPerspectiveTransform(src_quad, dst_quad) + # warp PDF page into place (transparent where outside) + warp = cv2.warpPerspective(src, M, (frame.shape[1], frame.shape[0])) + mask = cv2.warpPerspective(np.ones((h,w), dtype=np.uint8)*255, M, (frame.shape[1], frame.shape[0])) + # composite onto frame + inv = cv2.bitwise_not(mask) + bg = cv2.bitwise_and(frame, frame, mask=inv) + fg = cv2.bitwise_and(warp, warp, mask=mask) + np.copyto(frame, bg+fg) + +# hud.py +class HUD: + def __init__(self, marker_id, pdf_pages, default_page=0): + self.id = marker_id + self.page_index = default_page + self.pdf_pages = pdf_pages # list of preloaded page images + self.ref_quad = None # the 4-corner quad where to draw the PDF + self.last_seen = 0 # for timeout if arrow goes away + + def update(self, marker): + return + + def draw(self, frame): + return \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..22843e5 --- /dev/null +++ b/main.py @@ -0,0 +1,140 @@ +import cv2, numpy as np +from calibration import object_calibration +from hand_detection import detect_hands, check_hand +from helper import compute_distance +from hud import find_hud_placement +import math + +from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD +from config import DEBUG + +from hud import draw_hud_box, draw_pdf_page + +# ----- Global State ----- +measuring = False # measurement in progress +start_pt = None # measurement start point +calibrating = False # pinch-based calibration flag +object_calibrating = False # object calibration flag +fingertip_idx_global = None # last detected fingertip position + +hud_pos = None +hud_rot = None +register_hud = False + +# ----- Per-camera Processing ----- +def process_frame(frame): + global measuring, start_pt, calibrating + global object_calibrating + global fingertip_idx_global + + global register_hud, hud_pos, hud_rot + + frame_vis = cv2.flip(frame, 1) + + # Hand detection + results = detect_hands(frame_vis) + + fingertip_idx = None + if results.multi_hand_landmarks: + hand = results.multi_hand_landmarks[0] + fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating) + + fingertip_idx_global = fingertip_idx + + # object calibration + if object_calibrating and fingertip_idx is not None: + object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD) + object_calibrating = False + + # place hud + res = None + if hud_pos is None: + res = find_hud_placement(frame_vis) + else: + res = hud_pos, hud_rot + + + + # create proj output + projection_out = np.zeros_like(frame) + + # measuring + if measuring and start_pt and fingertip_idx_global: + cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2) + px = compute_distance(start_pt, fingertip_idx_global) + inch = px/PIXELS_PER_INCH; cm = inch*2.54 + mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2) + cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2) + + # hud + if res is not None or hud_pos: + if hud_pos: + arrow_tip = hud_pos + dir_vec = hud_rot + else: + arrow_tip, dir_vec = res + + dx, dy = dir_vec + arrow_angle = math.atan2(dy, dx) # result in radians + quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400)) + draw_pdf_page(projection_out, page_index=0, dst_quad=quad) + + if register_hud: + hud_pos = arrow_tip + hud_rot = dir_vec + register_hud = False + + + mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0 + frame_vis[mask] = projection_out[mask] + + return frame_vis, projection_out + +# ----- Main ----- ------------------------------------------------------------------------------------- +def main(): + all_captures = [] + for idx in CAMERA_INDICES: + capture = cv2.VideoCapture(idx) + capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH) + capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT) + all_captures.append(capture) + if not all(capture.isOpened() for capture in all_captures): + print("Error: could not open all cameras") + return + + print("Press 'o' for circle calib, 'q' to quit.") + while True: + frames = [capture.read()[1] for capture in all_captures] + frame = next((f for f in frames if f is not None), None) + + if frame is None: + break + + # process + projection, debug = process_frame(frame) + + # show windows + if DEBUG: + cv2.imshow('Debug Output', debug) + cv2.imshow('Projector Output', projection) + + key = cv2.waitKey(1) & 0xFF + + if key == ord('q'): + break + elif key == ord('o'): + global object_calibrating + object_calibrating = True + print("Entered circle calibration mode.") + elif key == ord('j'): + global register_hud, hud_pos + register_hud = True + hud_pos = None + + for capture in all_captures: + capture.release() + + cv2.destroyAllWindows() + +if __name__ == '__main__': + main() \ No newline at end of file