idk

Stuff/Untitled-1.txt

@@ -1,195 +1,150 @@
-import cv2
+import cv2, numpy as np, time, fitz
-import mediapipe as mp
+from calibration import object_calibration, draw_cal_circle
-import numpy as np
+from hand_detection import detect_hands, check_hand
+from helper import compute_distance
+from hud import find_hud_placement, draw_hud
+import math
 
-# ----- Helper Functions -----
+from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
-def compute_distance(p1, p2):
+from config import DEBUG
-    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
 
-# ----- Configuration -----
+from hud import draw_hud_box, draw_pdf_page
-CAMERA_INDICES = [0]          # List of camera device indices
-FRAME_WIDTH = 1280
-FRAME_HEIGHT = 720
-PIXELS_PER_INCH = 20          # will be set by calibration
-PINCH_THRESHOLD = 40          # px to start touch
-RELEASE_THRESHOLD = 60        # px to end touch
-CIRCLE_TOUCH_THRESHOLD = 20   # px tolerance for circle touch
 
-# ----- Initialize Hand Detector -----
-mp_hands = mp.solutions.hands
-mp_draw  = mp.solutions.drawing_utils
-hands = mp_hands.Hands(
-    static_image_mode=False,
-    max_num_hands=1,
-    min_detection_confidence=0.7,
-    min_tracking_confidence=0.5
-)
 
 # ----- Global State -----
 measuring = False             # measurement in progress
 start_pt = None               # measurement start point
 calibrating = False           # pinch-based calibration flag
-cal_start = None              # pinch calibration start point
 object_calibrating = False    # object calibration flag
-cal_circle = None             # reference circle (x,y,r)
 fingertip_idx_global = None   # last detected fingertip position
 
+
+hud_pos = None
+hud_rot = None
+register_hud = False
+
 # ----- Per-camera Processing -----
 def process_frame(frame):
-    global measuring, start_pt, calibrating, cal_start, object_calibrating, cal_circle, PIXELS_PER_INCH, fingertip_idx_global
+    global measuring, start_pt, calibrating
+    global object_calibrating
+    global fingertip_idx_global
+    
+    global register_hud, hud_pos, hud_rot
+
     frame_out = cv2.flip(frame, 1)
-    h, w, _ = frame_out.shape
+
+    frame_vis = frame_out.copy()
+    projection_debug = frame_out.copy()
 
     # Hand detection
-    rgb = cv2.cvtColor(frame_out, cv2.COLOR_BGR2RGB)
+
-    rgb.flags.writeable = False
+    results = detect_hands(frame_out)
-    results = hands.process(rgb)
-    rgb.flags.writeable = True
-    frame_vis = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
 
     fingertip_idx = None
-    fingertip_mid = None
-
     if results.multi_hand_landmarks:
         hand = results.multi_hand_landmarks[0]
-        mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS)
+        fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating)
-        # get index and middle finger tips and PIP joints
-        idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
-        idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]
-        mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
-        mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP]
-        ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
-        mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
-        fingertip_idx = (ix, iy)
-        fingertip_mid = (mx, my)
-        fingertip_idx_global = fingertip_idx
-        # draw fingertips
-        cv2.circle(frame_vis, fingertip_idx, 8, (0,255,0), -1)
-        cv2.circle(frame_vis, fingertip_mid,  8, (0,255,0), -1)
-        # check extension
-        index_ext = idx_tip.y < idx_pip.y
-        middle_ext = mid_tip.y < mid_pip.y
-        # if measurement in progress but fingers no longer both extended, stop measuring
-        if measuring and not (index_ext and middle_ext):
-            measuring = False
-        # pinch distance
-        pinch = compute_distance(fingertip_idx, fingertip_mid)
-        cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2)
-
-        # pinch calibration
-        if calibrating:
-            if pinch < PINCH_THRESHOLD and cal_start is None:
-                cal_start = fingertip_idx
-                print("Pinch calibration start set")
-            elif pinch > RELEASE_THRESHOLD and cal_start is not None:
-                cal_end = fingertip_idx
-                px = compute_distance(cal_start, cal_end)
-                inches = float(input("Enter actual distance between points (inches): "))
-                PIXELS_PER_INCH = px / inches
-                print(f"Calibrated: {PIXELS_PER_INCH:.2f} px/inch")
-                calibrating = False
-                cal_start = None
-        # measurement gesture
-        elif not object_calibrating and index_ext and middle_ext:
-            if pinch < PINCH_THRESHOLD and not measuring:
-                measuring = True
-                start_pt = fingertip_idx
-            elif pinch > RELEASE_THRESHOLD and measuring:
-                measuring = False
 
+    fingertip_idx_global = fingertip_idx
+ 
     # object calibration
     if object_calibrating and fingertip_idx is not None:
-        hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
+        object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
-        mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255]))
+        object_calibrating = False
-        masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask)
+    
-        gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
-        gray = cv2.medianBlur(gray,5)
-        circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100,
-                                   param1=50, param2=30, minRadius=10, maxRadius=300)
-        if circles is not None:
-            circles = np.round(circles[0]).astype(int)
-            touched = [(x,y,r) for x,y,r in circles
-                       if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD]
-            if touched:
-                touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2]))
-                x,y,r = touched[0]
-                cal_circle = (x,y,r)
-                PIXELS_PER_INCH = 2 * r
-                print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch")
-                object_calibrating = False
 
-    # permanent reference circle
+    res = None
-    if cal_circle:
+    if hud_pos is None:
-        cx,cy,cr = cal_circle
+        res = find_hud_placement(frame_vis)
-        cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2)
+    else:
-        cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1)
+        res = hud_pos, hud_rot
-        cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)
+
+    if res is not None:
+        arrow_tip, dir_vec = res
+        draw_hud(frame_vis, arrow_tip, dir_vec)
+
+        dx, dy = dir_vec
+        arrow_angle = math.atan2(dy, dx)   # result in radians
+        quad = draw_hud_box(frame_vis, arrow_tip, arrow_angle, size=(400,400))
+        draw_pdf_page(frame_vis, page_index=0, dst_quad=quad)
+
+        if register_hud:
+            hud_pos = arrow_tip
+            hud_rot = dir_vec
+            register_hud = False
 
     return frame_vis
 
-# ----- Main -----
+# ----- Main ----- -------------------------------------------------------------------------------------
 def main():
-    caps = []
+    global register_hud, hud_pos, hud_rot
+
+    all_captures = []
     for idx in CAMERA_INDICES:
-        cap = cv2.VideoCapture(idx)
+        capture = cv2.VideoCapture(idx)
-        cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
+        capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
-        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
+        capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
-        caps.append(cap)
+        all_captures.append(capture)
-    if not all(cap.isOpened() for cap in caps):
+    if not all(capture.isOpened() for capture in all_captures):
         print("Error: could not open all cameras")
         return
 
-    print("Press 'c' for pinch calib, 'o' for circle calib, 'q' to quit.")
+    print("Press 'o' for circle calib, 'q' to quit.")
     while True:
-        frames = [cap.read()[1] for cap in caps]
+        frames = [capture.read()[1] for capture in all_captures]
         frame = next((f for f in frames if f is not None), None)
+
         if frame is None:
             break
+
         # process
         full_view = process_frame(frame)
+
         # create proj output
-        proj = np.zeros_like(full_view)
+        projection_out = np.zeros_like(full_view)
-        if cal_circle:
+        
-            cx,cy,cr = cal_circle
-            cv2.circle(proj,(cx,cy),cr,(0,0,255),2)
         if measuring and start_pt and fingertip_idx_global:
-            cv2.line(proj, start_pt, fingertip_idx_global, (255,0,0),2)
+            cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
             px = compute_distance(start_pt, fingertip_idx_global)
             inch = px/PIXELS_PER_INCH; cm = inch*2.54
             mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
-            cv2.putText(proj,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
+            cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-                        cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
+            cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
+
+        if hud_pos:
+            dx, dy = hud_rot
+            arrow_angle = math.atan2(dy, dx)   # result in radians
+            quad = draw_hud_box(projection_out, hud_pos, arrow_angle, size=(400,400))
+            draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
+            
         # overlay proj onto debug
         debug = full_view.copy()
-        # overlay ref circle
+
-        if cal_circle:
-            cx,cy,cr = cal_circle
-            cv2.circle(debug,(cx,cy),cr,(0,0,255),2)
         # overlay measurement
         if measuring and start_pt and fingertip_idx_global:
-            cv2.line(debug, start_pt, fingertip_idx_global, (255,0,0),2)
+            cv2.line(debug, start_pt, fingertip_idx_global, (255,255,0),2)
             cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-                        cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
+            cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
 
         # show windows
-        cv2.imshow('Hand Measure', debug)
+        cv2.imshow('Debug Output', debug)
-        cv2.imshow('Projector Output', proj)
+        cv2.imshow('Projector Output', projection_out)
 
         key = cv2.waitKey(1) & 0xFF
+
         if key == ord('q'):
             break
-        elif key == ord('c'):
-            global calibrating
-            calibrating = True
-            cal_start = None
-            print("Entered pinch calibration mode.")
         elif key == ord('o'):
             global object_calibrating
             object_calibrating = True
             print("Entered circle calibration mode.")
+        elif key == ord('j'):
+            register_hud = True
+            hud_pos = None
+
+    for capture in all_captures:
+        capture.release()
 
-    for cap in caps:
-        cap.release()
     cv2.destroyAllWindows()
 
 if __name__ == '__main__':

TableIdk.py

@@ -1,279 +0,0 @@
-import cv2
-import mediapipe as mp
-import numpy as np
-
-# ----- Helper Functions -----
-def compute_distance(p1, p2):
-    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
-
-# ----- Configuration -----
-CAMERA_INDICES = [0]          # List of camera device indices
-FRAME_WIDTH = 1280
-FRAME_HEIGHT = 720
-PIXELS_PER_INCH = 38          # will be set by calibration
-PINCH_THRESHOLD = 40          # px to start touch
-RELEASE_THRESHOLD = 60        # px to end touch
-CIRCLE_TOUCH_THRESHOLD = 20   # px tolerance for circle touch
-
-# HSV range for shape color (tune for your arrow: now tailored for orange)
-LOWER_SHAPE = np.array([10, 100, 100])  # hue from 10° (orange) to
-UPPER_SHAPE = np.array([30, 255, 255])  # hue up to 30°, full sat/val range
-
-# dynamic list of registered arrows
-# each entry will be {'lower': np.array, 'upper': np.array, 'tip': (x,y) or None}
-shape_ranges = []
-
-# ----- Initialize Hand Detector -----
-mp_hands = mp.solutions.hands
-mp_draw  = mp.solutions.drawing_utils
-hands = mp_hands.Hands(
-    static_image_mode=False,
-    max_num_hands=1,
-    min_detection_confidence=0.7,
-    min_tracking_confidence=0.5
-)
-
-# ----- Global State -----
-measuring = False             # measurement in progress
-start_pt = None               # measurement start point
-calibrating = False           # pinch-based calibration flag
-cal_start = None              # pinch calibration start point
-object_calibrating = False    # object calibration flag
-cal_circle = None             # reference circle (x,y,r)
-fingertip_idx_global = None   # last detected fingertip position
-arrow_tip = None              # detected arrow tip position
-
-# ----- Per-camera Processing -----
-def process_frame(frame):
-    global measuring, start_pt, calibrating, cal_start
-    global object_calibrating, cal_circle, PIXELS_PER_INCH
-    global fingertip_idx_global, arrow_tip
-    
-    frame_out = cv2.flip(frame, 1)
-    h, w, _ = frame_out.shape
-
-    # Hand detection
-    rgb = cv2.cvtColor(frame_out, cv2.COLOR_BGR2RGB)
-    rgb.flags.writeable = False
-    results = hands.process(rgb)
-    rgb.flags.writeable = True
-    frame_vis = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
-
-    fingertip_idx = None
-    fingertip_mid = None
-
-    if results.multi_hand_landmarks:
-        hand = results.multi_hand_landmarks[0]
-        mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS)
-        # get index and middle finger tips and PIP joints
-        idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
-        idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]
-        mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
-        mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP]
-        ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
-        mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
-        fingertip_idx = (ix, iy)
-        fingertip_mid = (mx, my)
-        fingertip_idx_global = fingertip_idx
-        # draw fingertips
-        cv2.circle(frame_vis, fingertip_idx, 8, (0,255,0), -1)
-        cv2.circle(frame_vis, fingertip_mid,  8, (0,255,0), -1)
-        # check extension
-        index_ext = idx_tip.y < idx_pip.y
-        middle_ext = mid_tip.y < mid_pip.y
-        # if measurement in progress but fingers no longer both extended, stop measuring
-        if measuring and not (index_ext and middle_ext):
-            measuring = False
-        # pinch distance
-        pinch = compute_distance(fingertip_idx, fingertip_mid)
-        cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2)
-
-        # pinch calibration
-        if calibrating:
-            if pinch < PINCH_THRESHOLD and cal_start is None:
-                cal_start = fingertip_idx
-                print("Pinch calibration start set")
-            elif pinch > RELEASE_THRESHOLD and cal_start is not None:
-                cal_end = fingertip_idx
-                px = compute_distance(cal_start, cal_end)
-                inches = float(input("Enter actual distance between points (inches): "))
-                PIXELS_PER_INCH = px / inches
-                print(f"Calibrated: {PIXELS_PER_INCH:.2f} px/inch")
-                calibrating = False
-                cal_start = None
-        # measurement gesture
-        elif not object_calibrating and index_ext and middle_ext:
-            if pinch < PINCH_THRESHOLD and not measuring:
-                measuring = True
-                start_pt = fingertip_idx
-            elif pinch > RELEASE_THRESHOLD and measuring:
-                measuring = False
-
-    
-
-    # object calibration
-    if object_calibrating and fingertip_idx is not None:
-        hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
-        mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255]))
-        masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask)
-        gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
-        gray = cv2.medianBlur(gray,5)
-        circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100,
-                                   param1=50, param2=30, minRadius=10, maxRadius=300)
-        if circles is not None:
-            circles = np.round(circles[0]).astype(int)
-            touched = [(x,y,r) for x,y,r in circles
-                       if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD]
-            if touched:
-                touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2]))
-                x,y,r = touched[0]
-                cal_circle = (x,y,r)
-                PIXELS_PER_INCH = 2 * r
-                print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch")
-                object_calibrating = False
-
-    # permanent reference circle
-    if cal_circle:
-        cx,cy,cr = cal_circle
-        cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2)
-        cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1)
-        cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)
-    
-    # ----- Arrow Shape Detection & HUD Placement -----
-    # 1) Build a clean orange mask
-    hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
-    mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE)
-    kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
-    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern)
-    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern)
-
-    # 2) Find and filter contours
-    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-    for cnt in contours:
-        area = cv2.contourArea(cnt)
-        if area < 1000:
-            continue
-
-        # Approximate to polygon and require exactly 5 corners
-        peri = cv2.arcLength(cnt, True)
-        approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)
-        if len(approx) != 5:
-            continue
-
-        pts = approx.reshape(-1,2)
-        centroid = np.mean(pts, axis=0)
-
-        # Find the arrow tip as the corner farthest from centroid
-        dists = [np.linalg.norm(pt - centroid) for pt in pts]
-        tip_pt = pts[int(np.argmax(dists))]
-        raw_tip = (int(tip_pt[0]), int(tip_pt[1]))
-
-        # 3) Smooth the tip over time
-        alpha = 0.2
-        if arrow_tip is None:
-            arrow_tip = raw_tip
-        else:
-            arrow_tip = (
-              int(alpha * raw_tip[0] + (1-alpha) * arrow_tip[0]),
-              int(alpha * raw_tip[1] + (1-alpha) * arrow_tip[1])
-            )
-
-        # Draw the arrow & tip
-        cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2)
-        cv2.circle(frame_vis, arrow_tip, 8, (0,255,255), -1)
-        cv2.putText(frame_vis, "Arrow Tip", arrow_tip,
-                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,255), 2)
-
-        # Place the HUD box in the arrow’s pointing direction
-        dir_vec = tip_pt - centroid
-        norm = np.linalg.norm(dir_vec)
-        if norm>0:
-            dir_unit = dir_vec / norm
-            offset = 50
-            hud_center = (int(arrow_tip[0] + dir_unit[0]*offset),
-                          int(arrow_tip[1] + dir_unit[1]*offset))
-
-            # Build and rotate a 120×60 HUD rectangle
-            w2, h2 = 60, 30
-            theta = np.arctan2(dir_unit[1], dir_unit[0])
-            R = np.array([[ np.cos(theta), -np.sin(theta)],
-                          [ np.sin(theta),  np.cos(theta)]])
-            corners = np.array([[-w2,-h2], [w2,-h2], [w2,h2], [-w2,h2]])
-            hud_pts = (corners @ R.T) + np.array(hud_center)
-            hud_pts = hud_pts.astype(int)
-
-            cv2.drawContours(frame_vis, [hud_pts], -1, (255,0,255), 2)
-            cv2.putText(frame_vis, "HUD", hud_center,
-                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255,0,255), 2)
-
-
-
-    return frame_vis
-
-# ----- Main -----
-def main():
-    caps = []
-    for idx in CAMERA_INDICES:
-        cap = cv2.VideoCapture(idx)
-        cap.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
-        cap.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
-        caps.append(cap)
-    if not all(cap.isOpened() for cap in caps):
-        print("Error: could not open all cameras")
-        return
-
-    print("Press 'c' for pinch calib, 'o' for circle calib, 'q' to quit.")
-    while True:
-        frames = [cap.read()[1] for cap in caps]
-        frame = next((f for f in frames if f is not None), None)
-        if frame is None:
-            break
-        # process
-        full_view = process_frame(frame)
-        # create proj output
-        proj = np.zeros_like(full_view)
-        if cal_circle:
-            cx,cy,cr = cal_circle
-            cv2.circle(proj,(cx,cy),cr,(0,0,255),2)
-        if measuring and start_pt and fingertip_idx_global:
-            cv2.line(proj, start_pt, fingertip_idx_global, (255,0,0),2)
-            px = compute_distance(start_pt, fingertip_idx_global)
-            inch = px/PIXELS_PER_INCH; cm = inch*2.54
-            mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
-            cv2.putText(proj,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-                        cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
-        # overlay proj onto debug
-        debug = full_view.copy()
-        # overlay ref circle
-        if cal_circle:
-            cx,cy,cr = cal_circle
-            cv2.circle(debug,(cx,cy),cr,(0,0,255),2)
-        # overlay measurement
-        if measuring and start_pt and fingertip_idx_global:
-            cv2.line(debug, start_pt, fingertip_idx_global, (255,0,0),2)
-            cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
-                        cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
-
-        # show windows
-        cv2.imshow('Hand Measure', debug)
-        cv2.imshow('Projector Output', proj)
-
-        key = cv2.waitKey(1) & 0xFF
-        if key == ord('q'):
-            break
-        elif key == ord('c'):
-            global calibrating
-            calibrating = True
-            cal_start = None
-            print("Entered pinch calibration mode.")
-        elif key == ord('o'):
-            global object_calibrating
-            object_calibrating = True
-            print("Entered circle calibration mode.")
-
-    for cap in caps:
-        cap.release()
-    cv2.destroyAllWindows()
-
-if __name__ == '__main__':
-    main()

calibration.py

@@ -0,0 +1,32 @@
+import cv2, numpy as np
+
+from helper import compute_distance
+
+cal_circle = None             # reference circle (x,y,r)
+
+def object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD):
+    global cal_circle
+    hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
+    mask = cv2.inRange(hsv, np.array([10,100,100]), np.array([25,255,255]))
+    masked = cv2.bitwise_and(frame_vis, frame_vis, mask=mask)
+    gray = cv2.cvtColor(masked, cv2.COLOR_BGR2GRAY)
+    gray = cv2.medianBlur(gray,5)
+    circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT, 1.2, 100,
+                                param1=50, param2=30, minRadius=10, maxRadius=300)
+    if circles is not None:
+        circles = np.round(circles[0]).astype(int)
+        touched = [(x,y,r) for x,y,r in circles
+                    if abs(compute_distance((x,y), fingertip_idx)-r) < CIRCLE_TOUCH_THRESHOLD]
+        if touched:
+            touched.sort(key=lambda c: abs(compute_distance((c[0],c[1]), fingertip_idx)-c[2]))
+            x,y,r = touched[0]
+            cal_circle = (x,y,r)
+            PIXELS_PER_INCH = 2 * r
+            print(f"Circle calib: {PIXELS_PER_INCH:.2f} px/inch")
+
+def draw_cal_circle(frame_vis):
+        global cal_circle
+        cx,cy,cr = cal_circle
+        cv2.circle(frame_vis,(cx,cy),cr,(0,0,255),2)
+        cv2.drawMarker(frame_vis,(cx,cy),(0,0,255),cv2.MARKER_TILTED_CROSS,15,1)
+        cv2.putText(frame_vis,f"Ref r={cr} px",(cx-cr,cy+cr+20),cv2.FONT_HERSHEY_SIMPLEX,0.5,(0,0,255),1)

config.py

@@ -0,0 +1,27 @@
+import numpy as np
+
+# ----- Configuration -----
+CAMERA_INDICES = [0]          # List of camera device indices
+FRAME_WIDTH = 3980
+FRAME_HEIGHT = 2560
+PIXELS_PER_INCH = 38          # will be set by calibration
+PINCH_THRESHOLD = 40          # px to start touch
+RELEASE_THRESHOLD = 60        # px to end touch
+CIRCLE_TOUCH_THRESHOLD = 20   # px tolerance for circle touch
+
+# HSV range for shape color (tune for your arrow: now tailored for orange)
+LOWER_SHAPE = np.array([10, 100, 100])  # hue from 10° (orange) to
+UPPER_SHAPE = np.array([30, 255, 255])  # hue up to 30°, full sat/val range
+
+DEBOUNCE_TIME   = 1.0  # seconds
+
+# ——— Constants for Tabs & Buttons ———
+TAB_W, TAB_H   = 40, 60
+TAB_X          = 600    # adjust this to your projector output width
+TAB_Y0         = 50
+TAB_GAP        = 10
+# left/right arrow buttons
+LEFT_BTN_TOP   = (TAB_X, TAB_Y0+2*(TAB_H+TAB_GAP)+20)
+RIGHT_BTN_TOP  = (TAB_X, LEFT_BTN_TOP[1]+TAB_H+TAB_GAP)
+
+DEBUG = True

hand_detection.py

@@ -0,0 +1,84 @@
+import mediapipe as mp
+import cv2, numpy as np, time, fitz
+from helper import compute_distance
+from config import PINCH_THRESHOLD, RELEASE_THRESHOLD
+
+# ----- Initialize Hand Detector -----
+mp_hands = mp.solutions.hands
+mp_draw  = mp.solutions.drawing_utils
+hands = mp_hands.Hands(
+    static_image_mode=False,
+    max_num_hands=2,
+    min_detection_confidence=0.7,
+    min_tracking_confidence=0.5
+)
+
+start_pt = None               # measurement start point
+measuring = False             # measurement in progress
+
+def detect_hands(frame):
+    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    rgb.flags.writeable = False
+    results = hands.process(rgb)
+
+    return results
+
+
+def check_hand(hand, frame_vis, object_calibrating):
+    h, w, _ = frame_vis.shape
+
+    fingertip_idx = None
+    fingertip_mid = None
+    
+    mp_draw.draw_landmarks(frame_vis, hand, mp_hands.HAND_CONNECTIONS)
+    # get index and middle finger tips and PIP joints
+    idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
+    mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
+    ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
+    mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
+    fingertip_idx = (ix, iy)
+    fingertip_mid = (mx, my)
+
+    # draw fingertips
+    cv2.circle(frame_vis, fingertip_idx, 8, (255,255,0), -1)
+    cv2.circle(frame_vis, fingertip_mid,  8, (0,255,0), -1)
+
+    measuring, start_pt = is_measuring(frame_vis, hand, object_calibrating)
+
+    return fingertip_idx, measuring, start_pt
+
+def is_measuring(frame_vis, hand, object_calibrating):
+    global measuring, start_pt
+    h, w, _ = frame_vis.shape
+
+    idx_tip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
+    idx_pip = hand.landmark[mp_hands.HandLandmark.INDEX_FINGER_PIP]
+    mid_tip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_TIP]
+    mid_pip = hand.landmark[mp_hands.HandLandmark.MIDDLE_FINGER_PIP]
+
+    ix, iy = int(idx_tip.x * w), int(idx_tip.y * h)
+    mx, my = int(mid_tip.x * w), int(mid_tip.y * h)
+    fingertip_idx = (ix, iy)
+    fingertip_mid = (mx, my)
+
+    # check extension
+    index_ext = idx_tip.y < idx_pip.y
+    middle_ext = mid_tip.y < mid_pip.y
+
+    # if measurement in progress but fingers no longer both extended, stop measuring
+    if measuring and not (index_ext and middle_ext):
+        measuring = False
+
+    # pinch distance
+    pinch = compute_distance(fingertip_idx, fingertip_mid)
+    cv2.putText(frame_vis, f"Pinch: {int(pinch)} px", (10,30), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0),2)
+
+    # measurement gesture
+    if not object_calibrating and index_ext and middle_ext:
+        if pinch < PINCH_THRESHOLD and not measuring:
+            measuring = True
+            start_pt = fingertip_idx
+        elif pinch > RELEASE_THRESHOLD and measuring:
+            measuring = False
+
+    return measuring, start_pt

helper.py

@@ -0,0 +1,6 @@
+import numpy as np
+
+# ----- Helper Functions -----
+def compute_distance(p1, p2):
+    return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
+

hud.py

@@ -0,0 +1,136 @@
+import cv2, numpy as np
+from config import LOWER_SHAPE, UPPER_SHAPE
+import cv2
+import numpy as np
+import time
+import fitz  # PyMuPDF for PDF rendering
+
+
+
+def find_hud_placement(frame_vis):
+    # ----- Arrow Shape Detection & HUD Placement -----
+    # 1) Build a clean orange mask
+    hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
+    mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE)
+    kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
+    mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern)
+    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern)
+
+    # 2) Find and filter contours
+    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+    for cnt in contours:
+        area = cv2.contourArea(cnt)
+        if area < 1000:
+            continue
+
+        # Approximate to polygon and require exactly 5 corners
+        peri = cv2.arcLength(cnt, True)
+        approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)
+        if len(approx) != 5:
+            continue
+
+        pts = approx.reshape(-1,2)
+        centroid = np.mean(pts, axis=0)
+
+        # Find the arrow tip as the corner farthest from centroid
+        dists = [np.linalg.norm(pt - centroid) for pt in pts]
+        tip_pt = pts[int(np.argmax(dists))]
+        arrow_tip = (int(tip_pt[0]), int(tip_pt[1]))
+
+        # Place the HUD box in the arrow’s pointing direction
+        dir_vec = tip_pt - centroid
+
+        #ToDo remove
+        cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2)
+
+        return arrow_tip, dir_vec
+
+# ——————————————————————————————————————————————
+# PDF loading / rasterization at import time
+# ——————————————————————————————————————————————
+PDF_PATH = "DarkAngles.pdf"   # change this to your file
+PDF_DPI  = 600           # controls resolution of rasterization
+
+_doc = fitz.open(PDF_PATH)
+_pdf_pages = []
+for page in _doc:
+    # render page to pixmap at desired zoom
+    zoom = PDF_DPI / 72.0
+    mat  = fitz.Matrix(zoom, zoom)
+    pix  = page.get_pixmap(matrix=mat, alpha=False)
+    # convert pixmap to ndarray
+    img = np.frombuffer(pix.samples, dtype=np.uint8)
+    img = img.reshape(pix.height, pix.width, pix.n) 
+    if pix.n == 4:
+        img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
+    _pdf_pages.append(img)
+_doc.close()
+
+# ——————————————————————————————————————————————
+# HUD drawing routines
+# ——————————————————————————————————————————————
+
+# rotated rectangle → 4 pts
+def _rect_to_pts(center, size, angle_rad):
+    cx, cy = center
+    w, h  = size
+    # local corners
+    pts = np.array([
+        [-w/2, -h/2],
+        [ w/2, -h/2],
+        [ w/2,  h/2],
+        [-w/2,  h/2],
+    ])
+    # rotation
+    c, s = np.cos(angle_rad), np.sin(angle_rad)
+    R = np.array([[c, -s],[s, c]])
+    pts = pts.dot(R.T)
+    pts += np.array([cx, cy])
+    return pts.astype(np.float32)
+
+def draw_hud_box(frame, tip, angle_rad, size=(120, 60), color=(255,0,255), thickness=2):
+    """Draw a rotated HUD rectangle at `tip` pointing along `angle_rad`."""
+    # compute box corners
+    pts = _rect_to_pts(center=tip, size=size, angle_rad=angle_rad)
+    cv2.drawContours(frame, [pts.astype(int)], -1, color, thickness)
+    # label
+    cv2.putText(frame, "HUD", (int(tip[0]+5), int(tip[1]+5)), cv2.FONT_HERSHEY_SIMPLEX, 0.6, color, 2)
+    return pts  # return the quad for PDF warping
+
+def draw_pdf_page(frame, page_index, dst_quad):
+    """
+    Warp PDF page image #page_index into the quadrilateral dst_quad.
+    dst_quad: 4×2 float32 array of destination corners in clock-wise order.
+    """
+    if page_index < 0 or page_index >= len(_pdf_pages):
+        return
+    src = _pdf_pages[page_index]
+    h, w = src.shape[:2]
+
+    # source corners (tl, tr, br, bl)
+    src_quad = np.array([[0,0], [w,0], [w,h], [0,h]], dtype=np.float32)
+    # compute homography
+    M = cv2.getPerspectiveTransform(src_quad, dst_quad)
+    # warp PDF page into place (transparent where outside)
+    warp = cv2.warpPerspective(src, M, (frame.shape[1], frame.shape[0]))
+    mask = cv2.warpPerspective(np.ones((h,w), dtype=np.uint8)*255, M, (frame.shape[1], frame.shape[0]))
+    # composite onto frame
+    inv = cv2.bitwise_not(mask)
+    bg  = cv2.bitwise_and(frame, frame, mask=inv)
+    fg  = cv2.bitwise_and(warp, warp, mask=mask)
+    np.copyto(frame, bg+fg)
+
+# hud.py
+class HUD:
+    def __init__(self, marker_id, pdf_pages, default_page=0):
+        self.id         = marker_id
+        self.page_index = default_page
+        self.pdf_pages  = pdf_pages    # list of preloaded page images
+        self.ref_quad   = None         # the 4-corner quad where to draw the PDF
+        self.last_seen  = 0            # for timeout if arrow goes away
+
+    def update(self, marker):
+        return
+
+    def draw(self, frame):
+        return

main.py

@@ -0,0 +1,140 @@
+import cv2, numpy as np
+from calibration import object_calibration
+from hand_detection import detect_hands, check_hand
+from helper import compute_distance
+from hud import find_hud_placement
+import math
+
+from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
+from config import DEBUG
+
+from hud import draw_hud_box, draw_pdf_page
+
+# ----- Global State -----
+measuring = False             # measurement in progress
+start_pt = None               # measurement start point
+calibrating = False           # pinch-based calibration flag
+object_calibrating = False    # object calibration flag
+fingertip_idx_global = None   # last detected fingertip position
+
+hud_pos = None
+hud_rot = None
+register_hud = False
+
+# ----- Per-camera Processing -----
+def process_frame(frame):
+    global measuring, start_pt, calibrating
+    global object_calibrating
+    global fingertip_idx_global
+    
+    global register_hud, hud_pos, hud_rot
+
+    frame_vis = cv2.flip(frame, 1)
+
+    # Hand detection
+    results = detect_hands(frame_vis)
+
+    fingertip_idx = None
+    if results.multi_hand_landmarks:
+        hand = results.multi_hand_landmarks[0]
+        fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating)
+
+    fingertip_idx_global = fingertip_idx
+ 
+    # object calibration
+    if object_calibrating and fingertip_idx is not None:
+        object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
+        object_calibrating = False
+    
+    # place hud
+    res = None
+    if hud_pos is None:
+        res = find_hud_placement(frame_vis)
+    else:
+        res = hud_pos, hud_rot
+
+
+
+    # create proj output
+    projection_out = np.zeros_like(frame)
+    
+    # measuring
+    if measuring and start_pt and fingertip_idx_global:
+        cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
+        px = compute_distance(start_pt, fingertip_idx_global)
+        inch = px/PIXELS_PER_INCH; cm = inch*2.54
+        mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
+        cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
+
+    # hud
+    if res is not None or hud_pos:
+        if hud_pos:
+            arrow_tip = hud_pos
+            dir_vec = hud_rot
+        else:
+            arrow_tip, dir_vec = res
+
+        dx, dy = dir_vec
+        arrow_angle = math.atan2(dy, dx)   # result in radians
+        quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400))
+        draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
+
+        if register_hud:
+            hud_pos = arrow_tip
+            hud_rot = dir_vec
+            register_hud = False
+
+    
+    mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0
+    frame_vis[mask] = projection_out[mask]
+
+    return frame_vis, projection_out
+
+# ----- Main ----- -------------------------------------------------------------------------------------
+def main():
+    all_captures = []
+    for idx in CAMERA_INDICES:
+        capture = cv2.VideoCapture(idx)
+        capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
+        capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
+        all_captures.append(capture)
+    if not all(capture.isOpened() for capture in all_captures):
+        print("Error: could not open all cameras")
+        return
+
+    print("Press 'o' for circle calib, 'q' to quit.")
+    while True:
+        frames = [capture.read()[1] for capture in all_captures]
+        frame = next((f for f in frames if f is not None), None)
+
+        if frame is None:
+            break
+
+        # process
+        projection, debug = process_frame(frame)
+
+        # show windows
+        if DEBUG:
+            cv2.imshow('Debug Output', debug)
+        cv2.imshow('Projector Output', projection)
+
+        key = cv2.waitKey(1) & 0xFF
+
+        if key == ord('q'):
+            break
+        elif key == ord('o'):
+            global object_calibrating
+            object_calibrating = True
+            print("Entered circle calibration mode.")
+        elif key == ord('j'):
+            global register_hud, hud_pos
+            register_hud = True
+            hud_pos = None
+
+    for capture in all_captures:
+        capture.release()
+
+    cv2.destroyAllWindows()
+
+if __name__ == '__main__':
+    main()