Commit for Gitea
This commit is contained in:
DuOtto
Vendored
+12
@@ -0,0 +1,12 @@
|
|||||||
|
{
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Run main.py",
|
||||||
|
"type": "python",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "${workspaceFolder}/source/main.py",
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+65
-69
@@ -1,15 +1,16 @@
|
|||||||
import cv2, numpy as np, time, fitz
|
import cv2, numpy as np
|
||||||
from calibration import object_calibration, draw_cal_circle
|
from calibration import object_calibration
|
||||||
from hand_detection import detect_hands, check_hand
|
from hand_detection import detect_hands, check_hand
|
||||||
from helper import compute_distance
|
from helper import compute_distance
|
||||||
from hud import find_hud_placement, draw_hud
|
from source.UI.hud import find_hud_placement
|
||||||
import math
|
import math
|
||||||
|
|
||||||
from config import CAMERA_INDICES, FRAME_WIDTH, FRAME_HEIGHT, PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
|
from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
|
||||||
|
from config import FRAME_INTERVAL, TARGET_FPS
|
||||||
from config import DEBUG
|
from config import DEBUG
|
||||||
|
from setup import InitCameras
|
||||||
from hud import draw_hud_box, draw_pdf_page
|
#from hud import draw_hud_box, draw_pdf_page
|
||||||
|
import time
|
||||||
|
|
||||||
# ----- Global State -----
|
# ----- Global State -----
|
||||||
measuring = False # measurement in progress
|
measuring = False # measurement in progress
|
||||||
@@ -18,27 +19,22 @@ calibrating = False # pinch-based calibration flag
|
|||||||
object_calibrating = False # object calibration flag
|
object_calibrating = False # object calibration flag
|
||||||
fingertip_idx_global = None # last detected fingertip position
|
fingertip_idx_global = None # last detected fingertip position
|
||||||
|
|
||||||
|
|
||||||
hud_pos = None
|
hud_pos = None
|
||||||
hud_rot = None
|
hud_rot = None
|
||||||
register_hud = False
|
register_hud = False
|
||||||
|
|
||||||
# ----- Per-camera Processing -----
|
# ----- Per-camera Processing -----
|
||||||
def process_frame(frame):
|
def process_frame(frame, camera_id: int = 0):
|
||||||
global measuring, start_pt, calibrating
|
global measuring, start_pt, calibrating
|
||||||
global object_calibrating
|
global object_calibrating
|
||||||
global fingertip_idx_global
|
global fingertip_idx_global
|
||||||
|
|
||||||
global register_hud, hud_pos, hud_rot
|
global register_hud, hud_pos, hud_rot
|
||||||
|
|
||||||
frame_out = cv2.flip(frame, 1)
|
frame_vis = cv2.flip(frame, 1)
|
||||||
|
|
||||||
frame_vis = frame_out.copy()
|
|
||||||
projection_debug = frame_out.copy()
|
|
||||||
|
|
||||||
# Hand detection
|
# Hand detection
|
||||||
|
results = detect_hands(frame_vis)
|
||||||
results = detect_hands(frame_out)
|
|
||||||
|
|
||||||
fingertip_idx = None
|
fingertip_idx = None
|
||||||
if results.multi_hand_landmarks:
|
if results.multi_hand_landmarks:
|
||||||
@@ -52,83 +48,74 @@ def process_frame(frame):
|
|||||||
object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
|
object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
|
||||||
object_calibrating = False
|
object_calibrating = False
|
||||||
|
|
||||||
|
# place hud
|
||||||
res = None
|
res = None
|
||||||
if hud_pos is None:
|
if hud_pos is None:
|
||||||
res = find_hud_placement(frame_vis)
|
res = find_hud_placement(frame_vis)
|
||||||
else:
|
else:
|
||||||
res = hud_pos, hud_rot
|
res = hud_pos, hud_rot
|
||||||
|
|
||||||
if res is not None:
|
# create proj output
|
||||||
arrow_tip, dir_vec = res
|
projection_out = np.zeros_like(frame)
|
||||||
draw_hud(frame_vis, arrow_tip, dir_vec)
|
|
||||||
|
# measuring
|
||||||
|
if measuring and start_pt and fingertip_idx_global:
|
||||||
|
cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
|
||||||
|
px = compute_distance(start_pt, fingertip_idx_global)
|
||||||
|
inch = px/PIXELS_PER_INCH; cm = inch*2.54
|
||||||
|
mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
|
||||||
|
cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
|
||||||
|
|
||||||
|
# hud
|
||||||
|
if res is not None or hud_pos:
|
||||||
|
if hud_pos:
|
||||||
|
arrow_tip = hud_pos
|
||||||
|
dir_vec = hud_rot
|
||||||
|
else:
|
||||||
|
arrow_tip, dir_vec = res
|
||||||
|
|
||||||
dx, dy = dir_vec
|
dx, dy = dir_vec
|
||||||
arrow_angle = math.atan2(dy, dx) # result in radians
|
arrow_angle = math.atan2(dy, dx) # result in radians
|
||||||
quad = draw_hud_box(frame_vis, arrow_tip, arrow_angle, size=(400,400))
|
#quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400))
|
||||||
draw_pdf_page(frame_vis, page_index=0, dst_quad=quad)
|
#draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
|
||||||
|
|
||||||
if register_hud:
|
if register_hud:
|
||||||
hud_pos = arrow_tip
|
hud_pos = arrow_tip
|
||||||
hud_rot = dir_vec
|
hud_rot = dir_vec
|
||||||
register_hud = False
|
register_hud = False
|
||||||
|
|
||||||
return frame_vis
|
|
||||||
|
mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0
|
||||||
|
frame_vis[mask] = projection_out[mask]
|
||||||
|
|
||||||
|
return frame_vis, projection_out
|
||||||
|
|
||||||
# ----- Main ----- -------------------------------------------------------------------------------------
|
# ----- Main ----- -------------------------------------------------------------------------------------
|
||||||
def main():
|
def main():
|
||||||
global register_hud, hud_pos, hud_rot
|
all_captures = InitCameras()
|
||||||
|
|
||||||
all_captures = []
|
|
||||||
for idx in CAMERA_INDICES:
|
|
||||||
capture = cv2.VideoCapture(idx)
|
|
||||||
capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
|
|
||||||
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
|
|
||||||
all_captures.append(capture)
|
|
||||||
if not all(capture.isOpened() for capture in all_captures):
|
|
||||||
print("Error: could not open all cameras")
|
|
||||||
return
|
|
||||||
|
|
||||||
|
print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)")
|
||||||
print("Press 'o' for circle calib, 'q' to quit.")
|
print("Press 'o' for circle calib, 'q' to quit.")
|
||||||
while True:
|
|
||||||
frames = [capture.read()[1] for capture in all_captures]
|
|
||||||
frame = next((f for f in frames if f is not None), None)
|
|
||||||
|
|
||||||
if frame is None:
|
last_time = time.time()
|
||||||
|
while True:
|
||||||
|
processed_frames = []
|
||||||
|
|
||||||
|
for camera_id, capture in enumerate(all_captures):
|
||||||
|
ok, frame = capture.read()
|
||||||
|
if not ok or frame is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
projection, debug = process_frame(frame, camera_id)
|
||||||
|
processed_frames.append((camera_id, projection, debug))
|
||||||
|
|
||||||
|
if not processed_frames:
|
||||||
break
|
break
|
||||||
|
|
||||||
# process
|
for camera_id, projection, debug in processed_frames:
|
||||||
full_view = process_frame(frame)
|
if DEBUG:
|
||||||
|
cv2.imshow(f'Debug Output (cam {camera_id})', debug)
|
||||||
# create proj output
|
cv2.imshow(f'Projector Output (cam {camera_id})', projection)
|
||||||
projection_out = np.zeros_like(full_view)
|
|
||||||
|
|
||||||
if measuring and start_pt and fingertip_idx_global:
|
|
||||||
cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
|
|
||||||
px = compute_distance(start_pt, fingertip_idx_global)
|
|
||||||
inch = px/PIXELS_PER_INCH; cm = inch*2.54
|
|
||||||
mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
|
|
||||||
cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
|
|
||||||
cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
|
|
||||||
|
|
||||||
if hud_pos:
|
|
||||||
dx, dy = hud_rot
|
|
||||||
arrow_angle = math.atan2(dy, dx) # result in radians
|
|
||||||
quad = draw_hud_box(projection_out, hud_pos, arrow_angle, size=(400,400))
|
|
||||||
draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
|
|
||||||
|
|
||||||
# overlay proj onto debug
|
|
||||||
debug = full_view.copy()
|
|
||||||
|
|
||||||
# overlay measurement
|
|
||||||
if measuring and start_pt and fingertip_idx_global:
|
|
||||||
cv2.line(debug, start_pt, fingertip_idx_global, (255,255,0),2)
|
|
||||||
cv2.putText(debug,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10),
|
|
||||||
cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,0),2)
|
|
||||||
|
|
||||||
# show windows
|
|
||||||
cv2.imshow('Debug Output', debug)
|
|
||||||
cv2.imshow('Projector Output', projection_out)
|
|
||||||
|
|
||||||
key = cv2.waitKey(1) & 0xFF
|
key = cv2.waitKey(1) & 0xFF
|
||||||
|
|
||||||
@@ -139,9 +126,18 @@ def main():
|
|||||||
object_calibrating = True
|
object_calibrating = True
|
||||||
print("Entered circle calibration mode.")
|
print("Entered circle calibration mode.")
|
||||||
elif key == ord('j'):
|
elif key == ord('j'):
|
||||||
|
global register_hud, hud_pos
|
||||||
register_hud = True
|
register_hud = True
|
||||||
hud_pos = None
|
hud_pos = None
|
||||||
|
|
||||||
|
# ——— frame‐rate limiting ———
|
||||||
|
now = time.time()
|
||||||
|
elapsed = now - last_time
|
||||||
|
to_wait = FRAME_INTERVAL - elapsed
|
||||||
|
if to_wait > 0:
|
||||||
|
time.sleep(to_wait)
|
||||||
|
last_time = time.time()
|
||||||
|
|
||||||
for capture in all_captures:
|
for capture in all_captures:
|
||||||
capture.release()
|
capture.release()
|
||||||
|
|
||||||
|
|||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
{}
|
||||||
Vendored
+1
@@ -0,0 +1 @@
|
|||||||
|
{}
|
||||||
Vendored
+33
@@ -0,0 +1,33 @@
|
|||||||
|
{
|
||||||
|
"file-explorer": true,
|
||||||
|
"global-search": true,
|
||||||
|
"switcher": true,
|
||||||
|
"graph": true,
|
||||||
|
"backlink": true,
|
||||||
|
"canvas": true,
|
||||||
|
"outgoing-link": true,
|
||||||
|
"tag-pane": true,
|
||||||
|
"footnotes": false,
|
||||||
|
"properties": false,
|
||||||
|
"page-preview": true,
|
||||||
|
"daily-notes": true,
|
||||||
|
"templates": true,
|
||||||
|
"note-composer": true,
|
||||||
|
"command-palette": true,
|
||||||
|
"slash-command": false,
|
||||||
|
"editor-status": true,
|
||||||
|
"bookmarks": true,
|
||||||
|
"markdown-importer": false,
|
||||||
|
"zk-prefixer": false,
|
||||||
|
"random-note": false,
|
||||||
|
"outline": true,
|
||||||
|
"word-count": true,
|
||||||
|
"slides": false,
|
||||||
|
"audio-recorder": false,
|
||||||
|
"workspaces": false,
|
||||||
|
"file-recovery": true,
|
||||||
|
"publish": false,
|
||||||
|
"sync": true,
|
||||||
|
"bases": true,
|
||||||
|
"webviewer": false
|
||||||
|
}
|
||||||
Vendored
+22
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"collapse-filter": true,
|
||||||
|
"search": "",
|
||||||
|
"showTags": false,
|
||||||
|
"showAttachments": false,
|
||||||
|
"hideUnresolved": false,
|
||||||
|
"showOrphans": true,
|
||||||
|
"collapse-color-groups": true,
|
||||||
|
"colorGroups": [],
|
||||||
|
"collapse-display": true,
|
||||||
|
"showArrow": false,
|
||||||
|
"textFadeMultiplier": 0,
|
||||||
|
"nodeSizeMultiplier": 1,
|
||||||
|
"lineSizeMultiplier": 1,
|
||||||
|
"collapse-forces": true,
|
||||||
|
"centerStrength": 0.518713248970312,
|
||||||
|
"repelStrength": 10,
|
||||||
|
"linkStrength": 1,
|
||||||
|
"linkDistance": 250,
|
||||||
|
"scale": 1,
|
||||||
|
"close": true
|
||||||
|
}
|
||||||
Vendored
+254
@@ -0,0 +1,254 @@
|
|||||||
|
{
|
||||||
|
"main": {
|
||||||
|
"id": "833985cf0bef0982",
|
||||||
|
"type": "split",
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "9d07647e9b05c8a5",
|
||||||
|
"type": "tabs",
|
||||||
|
"dimension": 50,
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "777c49eacd333b57",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "markdown",
|
||||||
|
"state": {
|
||||||
|
"file": "Table/Hand Tracking.md",
|
||||||
|
"mode": "source",
|
||||||
|
"source": false
|
||||||
|
},
|
||||||
|
"icon": "lucide-file",
|
||||||
|
"title": "Hand Tracking"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "ec54e744a9373b63",
|
||||||
|
"type": "tabs",
|
||||||
|
"dimension": 50,
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "55f84d8c4a6e6b37",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "graph",
|
||||||
|
"state": {},
|
||||||
|
"icon": "lucide-git-fork",
|
||||||
|
"title": "Graph view"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "cdcef64cb6388441",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "release-notes",
|
||||||
|
"state": {
|
||||||
|
"currentVersion": "1.10.6"
|
||||||
|
},
|
||||||
|
"icon": "lucide-book-up",
|
||||||
|
"title": "Release Notes 1.10.6"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "48db892d028c7efb",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "markdown",
|
||||||
|
"state": {
|
||||||
|
"file": "Table/Addons/USB Camera Case.md",
|
||||||
|
"mode": "source",
|
||||||
|
"source": false
|
||||||
|
},
|
||||||
|
"icon": "lucide-file",
|
||||||
|
"title": "USB Camera Case"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "a871b6f7e2ef2b0b",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "markdown",
|
||||||
|
"state": {
|
||||||
|
"file": "Table/General.md",
|
||||||
|
"mode": "source",
|
||||||
|
"source": false
|
||||||
|
},
|
||||||
|
"icon": "lucide-file",
|
||||||
|
"title": "General"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "ec002b0aa870cc5d",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "release-notes",
|
||||||
|
"state": {
|
||||||
|
"currentVersion": "1.11.7"
|
||||||
|
},
|
||||||
|
"icon": "lucide-book-up",
|
||||||
|
"title": "Release Notes 1.11.7"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"currentTab": 4
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"direction": "vertical"
|
||||||
|
},
|
||||||
|
"left": {
|
||||||
|
"id": "f6a806b37d1ae6eb",
|
||||||
|
"type": "split",
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "9148cc17fa29c8df",
|
||||||
|
"type": "tabs",
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "1c44bf34acded4f0",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "file-explorer",
|
||||||
|
"state": {
|
||||||
|
"sortOrder": "alphabetical",
|
||||||
|
"autoReveal": false
|
||||||
|
},
|
||||||
|
"icon": "lucide-folder-closed",
|
||||||
|
"title": "Files"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "85e41cb90d92a857",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "search",
|
||||||
|
"state": {
|
||||||
|
"query": "",
|
||||||
|
"matchingCase": false,
|
||||||
|
"explainSearch": false,
|
||||||
|
"collapseAll": false,
|
||||||
|
"extraContext": false,
|
||||||
|
"sortOrder": "alphabetical"
|
||||||
|
},
|
||||||
|
"icon": "lucide-search",
|
||||||
|
"title": "Search"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "8ada34bb1b02dff6",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "bookmarks",
|
||||||
|
"state": {},
|
||||||
|
"icon": "lucide-bookmark",
|
||||||
|
"title": "Bookmarks"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"direction": "horizontal",
|
||||||
|
"width": 300
|
||||||
|
},
|
||||||
|
"right": {
|
||||||
|
"id": "b343b576de17e5fa",
|
||||||
|
"type": "split",
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "a960a5dad3743e2e",
|
||||||
|
"type": "tabs",
|
||||||
|
"children": [
|
||||||
|
{
|
||||||
|
"id": "d06b72b904a43a4e",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "backlink",
|
||||||
|
"state": {
|
||||||
|
"collapseAll": false,
|
||||||
|
"extraContext": false,
|
||||||
|
"sortOrder": "alphabetical",
|
||||||
|
"showSearch": false,
|
||||||
|
"searchQuery": "",
|
||||||
|
"backlinkCollapsed": false,
|
||||||
|
"unlinkedCollapsed": true
|
||||||
|
},
|
||||||
|
"icon": "links-coming-in",
|
||||||
|
"title": "Backlinks"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "906e21e00a9b4d73",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "outgoing-link",
|
||||||
|
"state": {
|
||||||
|
"linksCollapsed": false,
|
||||||
|
"unlinkedCollapsed": true
|
||||||
|
},
|
||||||
|
"icon": "links-going-out",
|
||||||
|
"title": "Outgoing links"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "6c98ae458072df7c",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "tag",
|
||||||
|
"state": {
|
||||||
|
"sortOrder": "frequency",
|
||||||
|
"useHierarchy": true,
|
||||||
|
"showSearch": false,
|
||||||
|
"searchQuery": ""
|
||||||
|
},
|
||||||
|
"icon": "lucide-tags",
|
||||||
|
"title": "Tags"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "0ce33e15635006fc",
|
||||||
|
"type": "leaf",
|
||||||
|
"state": {
|
||||||
|
"type": "outline",
|
||||||
|
"state": {
|
||||||
|
"followCursor": false,
|
||||||
|
"showSearch": false,
|
||||||
|
"searchQuery": ""
|
||||||
|
},
|
||||||
|
"icon": "lucide-list",
|
||||||
|
"title": "Outline"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"direction": "horizontal",
|
||||||
|
"width": 300,
|
||||||
|
"collapsed": true
|
||||||
|
},
|
||||||
|
"left-ribbon": {
|
||||||
|
"hiddenItems": {
|
||||||
|
"switcher:Open quick switcher": false,
|
||||||
|
"graph:Open graph view": false,
|
||||||
|
"canvas:Create new canvas": false,
|
||||||
|
"daily-notes:Open today's daily note": false,
|
||||||
|
"templates:Insert template": false,
|
||||||
|
"command-palette:Open command palette": false,
|
||||||
|
"bases:Create new base": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"active": "ec002b0aa870cc5d",
|
||||||
|
"lastOpenFiles": [
|
||||||
|
"Table/Hand Tracking.md",
|
||||||
|
"Table/Addons/Addons.md",
|
||||||
|
"Table/Addons/Cable Holder.md",
|
||||||
|
"Table/Addons/Switch.md",
|
||||||
|
"Table/Addons/USB Camera Case.md",
|
||||||
|
"Table/Addons/Universal Mount.md",
|
||||||
|
"Untitled",
|
||||||
|
"Table/General.md",
|
||||||
|
"Table/Addons",
|
||||||
|
"Table/Addons",
|
||||||
|
"Table"
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
|
||||||
|
| Addon | Description |
|
||||||
|
| ---------------- | ----------------------- |
|
||||||
|
| [[Switch]] | A casing for the switch |
|
||||||
|
| [[Cable Holder]] | |
|
||||||
|
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
|
||||||
|
| Depth | 11 |
|
||||||
|
| ----- | ---- |
|
||||||
|
| Widht | 9.45 |
|
||||||
|
| Hight | 2.9 |
|
||||||
|
| | |
|
||||||
|
| | |
|
||||||
|
| | |
|
||||||
|
| | |
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
Gaming Table for Warhammer and Stuff.
|
||||||
|
|
||||||
|
[[Addons]]
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
[[Hand Tracking]]
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"type": "PDFContainer",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"contenttype": "PDF",
|
||||||
|
"source": "warhammer40000_core&key_quickstartguide_eng_24.09-s2afk26smk.pdf"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
FuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txtFuckThisShit.txt
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 2.0 MiB |
Binary file not shown.
Binary file not shown.
BIN
Binary file not shown.
|
After Width: | Height: | Size: 688 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 782 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 615 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 688 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 782 KiB |
BIN
Binary file not shown.
|
After Width: | Height: | Size: 615 KiB |
+163
@@ -0,0 +1,163 @@
|
|||||||
|
{
|
||||||
|
"version": 1,
|
||||||
|
"cameras": [
|
||||||
|
{
|
||||||
|
"index": 0,
|
||||||
|
"pxWidth": 2048,
|
||||||
|
"pxHeight": 1536,
|
||||||
|
"camera_matrix": [
|
||||||
|
[
|
||||||
|
1637.301826012801,
|
||||||
|
0.0,
|
||||||
|
1115.6922895709363
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
1637.3016923198538,
|
||||||
|
660.417851928144
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
0.0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"distortion_coefficients": [
|
||||||
|
[
|
||||||
|
0.1880726328811075,
|
||||||
|
-0.3711284739403174,
|
||||||
|
-0.019390902558209716,
|
||||||
|
0.009835323140375036,
|
||||||
|
0.42533165931899103
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"rotation_matrix_world_to_camera": [
|
||||||
|
[
|
||||||
|
1.0,
|
||||||
|
0.0,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
1.0,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
0.0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"translation_vector_world_to_camera": [
|
||||||
|
[
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"camera_projection_matrix": [
|
||||||
|
[
|
||||||
|
1637.301826012801,
|
||||||
|
0.0,
|
||||||
|
1115.6922895709363,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
1637.3016923198538,
|
||||||
|
660.417851928144,
|
||||||
|
0.0
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
0.0,
|
||||||
|
1.0,
|
||||||
|
0.0
|
||||||
|
]
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"index": 3,
|
||||||
|
"pxWidth": 2048,
|
||||||
|
"pxHeight": 1536,
|
||||||
|
"camera_matrix": [
|
||||||
|
[
|
||||||
|
1629.3953951276987,
|
||||||
|
0.0,
|
||||||
|
946.5260213900043
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
1608.3790414523598,
|
||||||
|
656.5060407239171
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.0,
|
||||||
|
0.0,
|
||||||
|
1.0
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"distortion_coefficients": [
|
||||||
|
[
|
||||||
|
0.19251720938106318,
|
||||||
|
-0.21522629593789278,
|
||||||
|
-0.02730155124857322,
|
||||||
|
-0.03890715094594556,
|
||||||
|
0.21680710398239586
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"rotation_matrix_world_to_camera": [
|
||||||
|
[
|
||||||
|
0.08276206632981453,
|
||||||
|
-0.8544623995610863,
|
||||||
|
0.5128785900319194
|
||||||
|
],
|
||||||
|
[
|
||||||
|
0.720047607579738,
|
||||||
|
0.40706667659536877,
|
||||||
|
0.5619859105211598
|
||||||
|
],
|
||||||
|
[
|
||||||
|
-0.6889716127646442,
|
||||||
|
0.3227858865283794,
|
||||||
|
0.6489432858598454
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"translation_vector_world_to_camera": [
|
||||||
|
[
|
||||||
|
-176.11272583763233
|
||||||
|
],
|
||||||
|
[
|
||||||
|
-564.9970879948509
|
||||||
|
],
|
||||||
|
[
|
||||||
|
471.9913794371197
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"camera_projection_matrix": [
|
||||||
|
[
|
||||||
|
-517.2774297117204,
|
||||||
|
-1086.7318582180455,
|
||||||
|
1449.923719330272,
|
||||||
|
159794.8580057718
|
||||||
|
],
|
||||||
|
[
|
||||||
|
705.7954552118755,
|
||||||
|
866.628395475963,
|
||||||
|
1329.9215473279708,
|
||||||
|
-598864.2830424493
|
||||||
|
],
|
||||||
|
[
|
||||||
|
-0.6889716127646442,
|
||||||
|
0.3227858865283794,
|
||||||
|
0.6489432858598454,
|
||||||
|
471.9913794371197
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
import socket, struct, time
|
||||||
|
|
||||||
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||||
|
sock.sendto(
|
||||||
|
struct.pack("<4sBIQB", b"HAND", 1, 123, int(time.time()*1000), 0),
|
||||||
|
("127.0.0.1", 9000)
|
||||||
|
)
|
||||||
|
print("sent")
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
@@ -1,21 +0,0 @@
|
|||||||
import numpy as np
|
|
||||||
import cv2
|
|
||||||
|
|
||||||
# ----- Helper Functions -----
|
|
||||||
def compute_distance(p1, p2):
|
|
||||||
return np.hypot(p2[0] - p1[0], p2[1] - p1[1])
|
|
||||||
|
|
||||||
# 4) compute straightness per finger
|
|
||||||
def finger_straight(i_tip, i_pip, i_mcp):
|
|
||||||
d1 = np.hypot(*(np.array(i_tip)-np.array(i_pip)))
|
|
||||||
d2 = np.hypot(*(np.array(i_pip)-np.array(i_mcp)))
|
|
||||||
return float(np.clip(d1/(d2+1e-3), 0, 1))
|
|
||||||
|
|
||||||
def yes_no():
|
|
||||||
while True:
|
|
||||||
key = cv2.waitKey(1) & 0xFF
|
|
||||||
|
|
||||||
if key == ord("y"):
|
|
||||||
return True
|
|
||||||
if key == ord("n"):
|
|
||||||
return False
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
import cv2, numpy as np
|
|
||||||
from config import LOWER_SHAPE, UPPER_SHAPE
|
|
||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
import time
|
|
||||||
import fitz # PyMuPDF for PDF rendering
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def find_hud_placement(frame_vis):
|
|
||||||
# ----- Arrow Shape Detection & HUD Placement -----
|
|
||||||
# 1) Build a clean orange mask
|
|
||||||
hsv = cv2.cvtColor(frame_vis, cv2.COLOR_BGR2HSV)
|
|
||||||
mask = cv2.inRange(hsv, LOWER_SHAPE, UPPER_SHAPE)
|
|
||||||
kern = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))
|
|
||||||
mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kern)
|
|
||||||
mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kern)
|
|
||||||
|
|
||||||
# 2) Find and filter contours
|
|
||||||
contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
|
|
||||||
for cnt in contours:
|
|
||||||
area = cv2.contourArea(cnt)
|
|
||||||
if area < 1000:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Approximate to polygon and require exactly 5 corners
|
|
||||||
peri = cv2.arcLength(cnt, True)
|
|
||||||
approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)
|
|
||||||
if len(approx) != 5:
|
|
||||||
continue
|
|
||||||
|
|
||||||
pts = approx.reshape(-1,2)
|
|
||||||
centroid = np.mean(pts, axis=0)
|
|
||||||
|
|
||||||
# Find the arrow tip as the corner farthest from centroid
|
|
||||||
dists = [np.linalg.norm(pt - centroid) for pt in pts]
|
|
||||||
tip_pt = pts[int(np.argmax(dists))]
|
|
||||||
arrow_tip = (int(tip_pt[0]), int(tip_pt[1]))
|
|
||||||
|
|
||||||
# Place the HUD box in the arrow’s pointing direction
|
|
||||||
dir_vec = tip_pt - centroid
|
|
||||||
|
|
||||||
#ToDo remove
|
|
||||||
cv2.drawContours(frame_vis, [pts], -1, (0,255,255), 2)
|
|
||||||
|
|
||||||
return arrow_tip, dir_vec
|
|
||||||
|
|
||||||
# hud.py
|
|
||||||
class HUD:
|
|
||||||
def __init__(self, marker_id, pdf_pages, default_page=0):
|
|
||||||
self.id = marker_id
|
|
||||||
self.page_index = default_page
|
|
||||||
self.pdf_pages = pdf_pages # list of preloaded page images
|
|
||||||
self.ref_quad = None # the 4-corner quad where to draw the PDF
|
|
||||||
self.last_seen = 0 # for timeout if arrow goes away
|
|
||||||
|
|
||||||
def update(self, marker):
|
|
||||||
return
|
|
||||||
|
|
||||||
def draw(self, frame):
|
|
||||||
return
|
|
||||||
@@ -1,147 +0,0 @@
|
|||||||
import cv2, numpy as np
|
|
||||||
from calibration import object_calibration
|
|
||||||
from hand_detection import detect_hands, check_hand
|
|
||||||
from helper import compute_distance
|
|
||||||
from hud import find_hud_placement
|
|
||||||
import math
|
|
||||||
|
|
||||||
from config import PIXELS_PER_INCH, CIRCLE_TOUCH_THRESHOLD
|
|
||||||
from config import FRAME_INTERVAL, TARGET_FPS
|
|
||||||
from config import DEBUG
|
|
||||||
from setup import InitCameras
|
|
||||||
#from hud import draw_hud_box, draw_pdf_page
|
|
||||||
import time
|
|
||||||
|
|
||||||
# ----- Global State -----
|
|
||||||
measuring = False # measurement in progress
|
|
||||||
start_pt = None # measurement start point
|
|
||||||
calibrating = False # pinch-based calibration flag
|
|
||||||
object_calibrating = False # object calibration flag
|
|
||||||
fingertip_idx_global = None # last detected fingertip position
|
|
||||||
|
|
||||||
hud_pos = None
|
|
||||||
hud_rot = None
|
|
||||||
register_hud = False
|
|
||||||
|
|
||||||
# ----- Per-camera Processing -----
|
|
||||||
def process_frame(frame, camera_id: int = 0):
|
|
||||||
global measuring, start_pt, calibrating
|
|
||||||
global object_calibrating
|
|
||||||
global fingertip_idx_global
|
|
||||||
|
|
||||||
global register_hud, hud_pos, hud_rot
|
|
||||||
|
|
||||||
frame_vis = cv2.flip(frame, 1)
|
|
||||||
|
|
||||||
# Hand detection
|
|
||||||
results = detect_hands(frame_vis)
|
|
||||||
|
|
||||||
fingertip_idx = None
|
|
||||||
if results.multi_hand_landmarks:
|
|
||||||
hand = results.multi_hand_landmarks[0]
|
|
||||||
fingertip_idx, measuring, start_pt = check_hand(hand, frame_vis, object_calibrating)
|
|
||||||
|
|
||||||
fingertip_idx_global = fingertip_idx
|
|
||||||
|
|
||||||
# object calibration
|
|
||||||
if object_calibrating and fingertip_idx is not None:
|
|
||||||
object_calibration(frame_vis, fingertip_idx, CIRCLE_TOUCH_THRESHOLD)
|
|
||||||
object_calibrating = False
|
|
||||||
|
|
||||||
# place hud
|
|
||||||
res = None
|
|
||||||
if hud_pos is None:
|
|
||||||
res = find_hud_placement(frame_vis)
|
|
||||||
else:
|
|
||||||
res = hud_pos, hud_rot
|
|
||||||
|
|
||||||
# create proj output
|
|
||||||
projection_out = np.zeros_like(frame)
|
|
||||||
|
|
||||||
# measuring
|
|
||||||
if measuring and start_pt and fingertip_idx_global:
|
|
||||||
cv2.line(projection_out, start_pt, fingertip_idx_global, (255,0,0),2)
|
|
||||||
px = compute_distance(start_pt, fingertip_idx_global)
|
|
||||||
inch = px/PIXELS_PER_INCH; cm = inch*2.54
|
|
||||||
mid = ((start_pt[0]+fingertip_idx_global[0])//2,(start_pt[1]+fingertip_idx_global[1])//2)
|
|
||||||
cv2.putText(projection_out,f"{inch:.2f}in/{cm:.1f}cm",(mid[0]+10,mid[1]-10), cv2.FONT_HERSHEY_SIMPLEX,0.7,(255,0,255),2)
|
|
||||||
|
|
||||||
# hud
|
|
||||||
if res is not None or hud_pos:
|
|
||||||
if hud_pos:
|
|
||||||
arrow_tip = hud_pos
|
|
||||||
dir_vec = hud_rot
|
|
||||||
else:
|
|
||||||
arrow_tip, dir_vec = res
|
|
||||||
|
|
||||||
dx, dy = dir_vec
|
|
||||||
arrow_angle = math.atan2(dy, dx) # result in radians
|
|
||||||
#quad = draw_hud_box(projection_out, arrow_tip, arrow_angle, size=(400,400))
|
|
||||||
#draw_pdf_page(projection_out, page_index=0, dst_quad=quad)
|
|
||||||
|
|
||||||
if register_hud:
|
|
||||||
hud_pos = arrow_tip
|
|
||||||
hud_rot = dir_vec
|
|
||||||
register_hud = False
|
|
||||||
|
|
||||||
|
|
||||||
mask = cv2.cvtColor(projection_out, cv2.COLOR_BGR2GRAY) > 0
|
|
||||||
frame_vis[mask] = projection_out[mask]
|
|
||||||
|
|
||||||
return frame_vis, projection_out
|
|
||||||
|
|
||||||
# ----- Main ----- -------------------------------------------------------------------------------------
|
|
||||||
def main():
|
|
||||||
all_captures = InitCameras()
|
|
||||||
|
|
||||||
print(f"Running at up to {TARGET_FPS} FPS (interval={FRAME_INTERVAL:.3f}s)")
|
|
||||||
print("Press 'o' for circle calib, 'q' to quit.")
|
|
||||||
|
|
||||||
last_time = time.time()
|
|
||||||
while True:
|
|
||||||
processed_frames = []
|
|
||||||
|
|
||||||
for camera_id, capture in enumerate(all_captures):
|
|
||||||
ok, frame = capture.read()
|
|
||||||
if not ok or frame is None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
projection, debug = process_frame(frame, camera_id)
|
|
||||||
processed_frames.append((camera_id, projection, debug))
|
|
||||||
|
|
||||||
if not processed_frames:
|
|
||||||
break
|
|
||||||
|
|
||||||
for camera_id, projection, debug in processed_frames:
|
|
||||||
if DEBUG:
|
|
||||||
cv2.imshow(f'Debug Output (cam {camera_id})', debug)
|
|
||||||
cv2.imshow(f'Projector Output (cam {camera_id})', projection)
|
|
||||||
|
|
||||||
key = cv2.waitKey(1) & 0xFF
|
|
||||||
|
|
||||||
if key == ord('q'):
|
|
||||||
break
|
|
||||||
elif key == ord('o'):
|
|
||||||
global object_calibrating
|
|
||||||
object_calibrating = True
|
|
||||||
print("Entered circle calibration mode.")
|
|
||||||
elif key == ord('j'):
|
|
||||||
global register_hud, hud_pos
|
|
||||||
register_hud = True
|
|
||||||
hud_pos = None
|
|
||||||
|
|
||||||
# ——— frame‐rate limiting ———
|
|
||||||
now = time.time()
|
|
||||||
elapsed = now - last_time
|
|
||||||
to_wait = FRAME_INTERVAL - elapsed
|
|
||||||
if to_wait > 0:
|
|
||||||
time.sleep(to_wait)
|
|
||||||
last_time = time.time()
|
|
||||||
|
|
||||||
for capture in all_captures:
|
|
||||||
capture.release()
|
|
||||||
|
|
||||||
cv2.destroyAllWindows()
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
import cv2
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
from config import CAMERA_INDICES_TO_CHECK, FRAME_WIDTH, FRAME_HEIGHT
|
|
||||||
|
|
||||||
def _preview_camera(capture: cv2.VideoCapture, cameraID: int) -> str:
|
|
||||||
"""Show a live preview and return the user's choice: 'add', 'skip', or 'quit'."""
|
|
||||||
window_name = f"Camera {cameraID}"
|
|
||||||
cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
|
|
||||||
print(f"[Camera {cameraID}] Press 'y' to add, 'n' to skip, 'q' to stop scanning.")
|
|
||||||
|
|
||||||
try:
|
|
||||||
while True:
|
|
||||||
ok, frame = capture.read()
|
|
||||||
if not ok:
|
|
||||||
print(f"[Camera {cameraID}] Failed to read frame; skipping.")
|
|
||||||
return "skip"
|
|
||||||
|
|
||||||
cv2.imshow(window_name, frame)
|
|
||||||
|
|
||||||
key = cv2.waitKey(1) & 0xFF
|
|
||||||
|
|
||||||
if key == ord("y"):
|
|
||||||
print(f"[Camera {cameraID}] Selected.")
|
|
||||||
return "add"
|
|
||||||
if key == ord("n"):
|
|
||||||
print(f"[Camera {cameraID}] Skipped.")
|
|
||||||
return "skip"
|
|
||||||
if key == ord("q") or key == 27: # 27 == ESC
|
|
||||||
print(f"[Camera {cameraID}] Stopping camera scan.")
|
|
||||||
return "quit"
|
|
||||||
finally:
|
|
||||||
cv2.destroyWindow(window_name)
|
|
||||||
|
|
||||||
|
|
||||||
def InitCameras():
|
|
||||||
allCaptures = []
|
|
||||||
|
|
||||||
setup_window = "Camera Setup"
|
|
||||||
instructions = np.zeros((240, 560, 3), dtype=np.uint8)
|
|
||||||
cv2.putText(instructions, "Camera Setup", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (0, 200, 255), 2)
|
|
||||||
cv2.putText(instructions, "Each camera will preview in its own window.", (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
|
|
||||||
cv2.putText(instructions, "Use 'y' to add, 'n' to skip, 'q'/ESC to finish.", (20, 125), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
|
|
||||||
cv2.putText(instructions, "Close this window when you're done.", (20, 160), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (200, 200, 200), 1)
|
|
||||||
|
|
||||||
cv2.namedWindow(setup_window, cv2.WINDOW_AUTOSIZE)
|
|
||||||
cv2.imshow(setup_window, instructions)
|
|
||||||
cv2.waitKey(1)
|
|
||||||
|
|
||||||
try:
|
|
||||||
for cameraID in range(CAMERA_INDICES_TO_CHECK):
|
|
||||||
capture = cv2.VideoCapture(cameraID)
|
|
||||||
|
|
||||||
if not capture.isOpened():
|
|
||||||
capture.release()
|
|
||||||
continue
|
|
||||||
|
|
||||||
capture.set(cv2.CAP_PROP_FRAME_WIDTH, FRAME_WIDTH)
|
|
||||||
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT)
|
|
||||||
|
|
||||||
decision = _preview_camera(capture, cameraID)
|
|
||||||
|
|
||||||
if decision == "add":
|
|
||||||
allCaptures.append(capture)
|
|
||||||
else:
|
|
||||||
capture.release()
|
|
||||||
|
|
||||||
if decision == "quit":
|
|
||||||
break
|
|
||||||
|
|
||||||
if not all(capture.isOpened() for capture in allCaptures):
|
|
||||||
print("Error: could not open all selected cameras.")
|
|
||||||
return
|
|
||||||
|
|
||||||
if not allCaptures:
|
|
||||||
print("No cameras selected.")
|
|
||||||
return
|
|
||||||
|
|
||||||
return allCaptures
|
|
||||||
finally:
|
|
||||||
cv2.destroyWindow(setup_window)
|
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
import cv2
|
||||||
|
print("OpenCV:", cv2.__version__)
|
||||||
|
print("has aruco:", hasattr(cv2, "aruco"))
|
||||||
|
print("has CharucoBoard:", hasattr(cv2.aruco, "CharucoBoard"))
|
||||||
|
print("has ArucoDetector:", hasattr(cv2.aruco, "ArucoDetector"))
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
# ---- Board parameters (CHANGE THESE ONLY IF YOU REPRINT) ----
|
||||||
|
squares_x = 6 # number of chessboard squares in X
|
||||||
|
squares_y = 9 # number of chessboard squares in Y
|
||||||
|
square_length = 25 # mm
|
||||||
|
marker_length = 18 # mm (must be < square_length)
|
||||||
|
dictionary_id = cv2.aruco.DICT_4X4_50
|
||||||
|
dpi = 300 # print DPI
|
||||||
|
# ------------------------------------------------------------
|
||||||
|
|
||||||
|
dictionary = cv2.aruco.getPredefinedDictionary(dictionary_id)
|
||||||
|
board = cv2.aruco.CharucoBoard(
|
||||||
|
(squares_x, squares_y),
|
||||||
|
square_length,
|
||||||
|
marker_length,
|
||||||
|
dictionary
|
||||||
|
)
|
||||||
|
|
||||||
|
# Convert physical size (mm) to pixels for printing
|
||||||
|
mm_to_inch = 1 / 25.4
|
||||||
|
width_mm = squares_x * square_length
|
||||||
|
height_mm = squares_y * square_length
|
||||||
|
|
||||||
|
width_px = int(width_mm * mm_to_inch * dpi)
|
||||||
|
height_px = int(height_mm * mm_to_inch * dpi)
|
||||||
|
|
||||||
|
img = board.generateImage((width_px, height_px))
|
||||||
|
img = 255 - img
|
||||||
|
cv2.imwrite("H:\Table\charuco_A4.png", img)
|
||||||
|
print("Saved charuco_A4.png")
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
"""
|
||||||
|
ChArUco detection confidence test (OpenCV)
|
||||||
|
|
||||||
|
What it does:
|
||||||
|
- Opens one or multiple cameras
|
||||||
|
- Detects ArUco markers and interpolated ChArUco corners
|
||||||
|
- Draws overlays
|
||||||
|
- Prints a simple "confidence" score per camera:
|
||||||
|
markers_found, charuco_corners_found, and a normalized confidence value
|
||||||
|
|
||||||
|
Requirements:
|
||||||
|
- OpenCV built with aruco module (opencv-contrib-python)
|
||||||
|
pip install opencv-contrib-python
|
||||||
|
"""
|
||||||
|
|
||||||
|
import cv2
|
||||||
|
import numpy as np
|
||||||
|
import time
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# USER SETTINGS (match your printed board!)
|
||||||
|
# ----------------------------
|
||||||
|
CAM_IDS = [0, 2] # set to [0] for single camera test, or [0,2,3,...]
|
||||||
|
USE_DSHOW_ON_WINDOWS = True # good for Windows
|
||||||
|
RESOLUTION = (3264, 2448) # (width, height) if your cameras support it; else set None
|
||||||
|
DICT_ID = cv2.aruco.DICT_4X4_50
|
||||||
|
|
||||||
|
SQUARES_X = 6 # number of chessboard squares in X
|
||||||
|
SQUARES_Y = 9 # number of chessboard squares in Y
|
||||||
|
SQUARE_LEN_MM = 25.0 # square size in mm
|
||||||
|
MARKER_LEN_MM = 18.0 # marker size in mm (must be < square)
|
||||||
|
|
||||||
|
# Confidence thresholds (tune if needed)
|
||||||
|
MIN_MARKERS_OK = 4 # markers to consider "good"
|
||||||
|
MIN_CHARUCO_OK = 15 # charuco corners to consider "good"
|
||||||
|
|
||||||
|
# Display
|
||||||
|
WINDOW_SCALE = 0.5 # downscale for display if res is huge (0.5 = half size)
|
||||||
|
PRINT_EVERY_SEC = 0.5
|
||||||
|
# ----------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def open_camera(cam_id: int):
|
||||||
|
backend = cv2.CAP_DSHOW if (USE_DSHOW_ON_WINDOWS and hasattr(cv2, "CAP_DSHOW")) else 0
|
||||||
|
cap = cv2.VideoCapture(cam_id, backend)
|
||||||
|
if not cap.isOpened():
|
||||||
|
return None
|
||||||
|
|
||||||
|
if RESOLUTION is not None:
|
||||||
|
w, h = RESOLUTION
|
||||||
|
cap.set(cv2.CAP_PROP_FRAME_WIDTH, float(w))
|
||||||
|
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, float(h))
|
||||||
|
|
||||||
|
return cap
|
||||||
|
|
||||||
|
|
||||||
|
def compute_confidence(markers: int, charuco: int) -> float:
|
||||||
|
"""
|
||||||
|
Simple normalized confidence heuristic:
|
||||||
|
- markers contribute up to MIN_MARKERS_OK
|
||||||
|
- charuco corners contribute up to MIN_CHARUCO_OK
|
||||||
|
"""
|
||||||
|
m = min(markers / max(MIN_MARKERS_OK, 1), 1.0)
|
||||||
|
c = min(charuco / max(MIN_CHARUCO_OK, 1), 1.0)
|
||||||
|
# Weighted: charuco corners matter more for calibration quality
|
||||||
|
return 0.35 * m + 0.65 * c
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
dictionary = cv2.aruco.getPredefinedDictionary(DICT_ID)
|
||||||
|
board = cv2.aruco.CharucoBoard(
|
||||||
|
(SQUARES_X, SQUARES_Y),
|
||||||
|
SQUARE_LEN_MM,
|
||||||
|
MARKER_LEN_MM,
|
||||||
|
dictionary
|
||||||
|
)
|
||||||
|
|
||||||
|
# Detector parameters
|
||||||
|
detector_params = cv2.aruco.DetectorParameters()
|
||||||
|
# You can tweak these if detection is unstable:
|
||||||
|
# detector_params.adaptiveThreshWinSizeMin = 3
|
||||||
|
# detector_params.adaptiveThreshWinSizeMax = 23
|
||||||
|
# detector_params.adaptiveThreshWinSizeStep = 10
|
||||||
|
|
||||||
|
detector = cv2.aruco.ArucoDetector(dictionary, detector_params)
|
||||||
|
|
||||||
|
caps = {}
|
||||||
|
for cam_id in CAM_IDS:
|
||||||
|
cap = open_camera(cam_id)
|
||||||
|
if cap is None:
|
||||||
|
print(f"[ERR] Could not open camera {cam_id}")
|
||||||
|
else:
|
||||||
|
caps[cam_id] = cap
|
||||||
|
print(f"[OK] Opened camera {cam_id}")
|
||||||
|
|
||||||
|
if not caps:
|
||||||
|
print("No cameras opened. Exiting.")
|
||||||
|
return
|
||||||
|
|
||||||
|
last_print = 0.0
|
||||||
|
|
||||||
|
print("\nControls:")
|
||||||
|
print(" ESC = quit")
|
||||||
|
print(" Space = print one-shot stats immediately\n")
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
frames_vis = []
|
||||||
|
stats = {}
|
||||||
|
|
||||||
|
for cam_id, cap in caps.items():
|
||||||
|
ok, frame = cap.read()
|
||||||
|
if not ok or frame is None:
|
||||||
|
stats[cam_id] = (0, 0, 0.0)
|
||||||
|
continue
|
||||||
|
|
||||||
|
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||||
|
gray = 255 - gray
|
||||||
|
# Detect markers
|
||||||
|
corners, ids, rejected = detector.detectMarkers(gray)
|
||||||
|
markers_found = 0 if ids is None else len(ids)
|
||||||
|
|
||||||
|
# Draw markers
|
||||||
|
vis = frame.copy()
|
||||||
|
if ids is not None:
|
||||||
|
cv2.aruco.drawDetectedMarkers(vis, corners, ids)
|
||||||
|
|
||||||
|
# Interpolate ChArUco corners (requires some markers found)
|
||||||
|
charuco_found = 0
|
||||||
|
if ids is not None and len(ids) > 0:
|
||||||
|
charuco_detector = cv2.aruco.CharucoDetector(board)
|
||||||
|
|
||||||
|
charuco_corners = None
|
||||||
|
charuco_ids = None
|
||||||
|
|
||||||
|
if ids is not None and len(ids) > 0:
|
||||||
|
res = charuco_detector.detectBoard(gray)
|
||||||
|
|
||||||
|
# OpenCV versions differ in what they return; handle both safely.
|
||||||
|
# Common patterns:
|
||||||
|
# (charucoCorners, charucoIds, markerCorners, markerIds)
|
||||||
|
# (charucoCorners, charucoIds, rejectedMarkerCandidates)
|
||||||
|
# (charucoCorners, charucoIds, ...)
|
||||||
|
charuco_corners = res[0] if len(res) > 0 else None
|
||||||
|
charuco_ids = res[1] if len(res) > 1 else None
|
||||||
|
charuco_found = 0
|
||||||
|
if charuco_ids is not None:
|
||||||
|
charuco_found = len(charuco_ids)
|
||||||
|
cv2.aruco.drawDetectedCornersCharuco(vis, charuco_corners, charuco_ids)
|
||||||
|
|
||||||
|
conf = compute_confidence(markers_found, charuco_found)
|
||||||
|
stats[cam_id] = (markers_found, charuco_found, conf)
|
||||||
|
|
||||||
|
# Overlay text
|
||||||
|
h, w = vis.shape[:2]
|
||||||
|
lines = [
|
||||||
|
f"Cam {cam_id}",
|
||||||
|
f"Markers: {markers_found}",
|
||||||
|
f"ChArUco corners: {charuco_found}",
|
||||||
|
f"Confidence: {conf:.2f}",
|
||||||
|
"GOOD" if (markers_found >= MIN_MARKERS_OK and charuco_found >= MIN_CHARUCO_OK) else "MOVE / LIGHT / FOCUS"
|
||||||
|
]
|
||||||
|
|
||||||
|
y = 30
|
||||||
|
for line in lines:
|
||||||
|
cv2.putText(vis, line, (20, y), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0) if "GOOD" in line else (0, 200, 255), 2)
|
||||||
|
y += 32
|
||||||
|
|
||||||
|
# Downscale for display
|
||||||
|
if WINDOW_SCALE != 1.0:
|
||||||
|
vis = cv2.resize(vis, None, fx=WINDOW_SCALE, fy=WINDOW_SCALE, interpolation=cv2.INTER_AREA)
|
||||||
|
|
||||||
|
frames_vis.append(vis)
|
||||||
|
|
||||||
|
# Combine displays
|
||||||
|
if frames_vis:
|
||||||
|
# Stack horizontally; if many cams, wrap to multiple rows
|
||||||
|
max_per_row = 3
|
||||||
|
rows = []
|
||||||
|
for i in range(0, len(frames_vis), max_per_row):
|
||||||
|
row = frames_vis[i:i + max_per_row]
|
||||||
|
# pad heights
|
||||||
|
max_h = max(img.shape[0] for img in row)
|
||||||
|
padded = []
|
||||||
|
for img in row:
|
||||||
|
if img.shape[0] < max_h:
|
||||||
|
pad = max_h - img.shape[0]
|
||||||
|
img = cv2.copyMakeBorder(img, 0, pad, 0, 0, cv2.BORDER_CONSTANT, value=(0, 0, 0))
|
||||||
|
padded.append(img)
|
||||||
|
rows.append(np.hstack(padded))
|
||||||
|
grid = np.vstack(rows)
|
||||||
|
|
||||||
|
cv2.imshow("ChArUco Confidence Test", grid)
|
||||||
|
|
||||||
|
key = cv2.waitKey(1) & 0xFF
|
||||||
|
now = time.time()
|
||||||
|
|
||||||
|
if key == 27: # ESC
|
||||||
|
break
|
||||||
|
|
||||||
|
if key == 32: # Space
|
||||||
|
last_print = 0 # force print now
|
||||||
|
|
||||||
|
if now - last_print >= PRINT_EVERY_SEC:
|
||||||
|
last_print = now
|
||||||
|
# Print compact stats
|
||||||
|
msg = " | ".join(
|
||||||
|
f"cam{cid}: M={m} C={c} conf={conf:.2f}"
|
||||||
|
for cid, (m, c, conf) in sorted(stats.items())
|
||||||
|
)
|
||||||
|
print(msg)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
for cap in caps.values():
|
||||||
|
cap.release()
|
||||||
|
cv2.destroyAllWindows()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import time
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
class HandUdpSender:
|
||||||
|
MAGIC = b'HAND'
|
||||||
|
VERSION = 2
|
||||||
|
|
||||||
|
def __init__(self, host="127.0.0.1", port=9000):
|
||||||
|
self.addr = (host, port)
|
||||||
|
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||||
|
self.seq = 0
|
||||||
|
|
||||||
|
def _encode_handedness(self, s: str) -> int:
|
||||||
|
if s == "Left":
|
||||||
|
return 1
|
||||||
|
if s == "Right":
|
||||||
|
return 2
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def send_hands3d(self, hands3d: list[dict]):
|
||||||
|
"""
|
||||||
|
hands3d: list of dicts:
|
||||||
|
{
|
||||||
|
"landmarks": (21,3) np.ndarray float,
|
||||||
|
"handedness": "Left"/"Right"/"Unknown",
|
||||||
|
"confidence": float (0..1)
|
||||||
|
}
|
||||||
|
Sends one UDP datagram containing all hands.
|
||||||
|
"""
|
||||||
|
self.seq += 1
|
||||||
|
ts_ms = int(time.time() * 1000)
|
||||||
|
|
||||||
|
hand_count = min(len(hands3d), 255)
|
||||||
|
|
||||||
|
# Header: magic(4s), version(B), seq(I), ts(Q), hand_count(B)
|
||||||
|
packet = bytearray()
|
||||||
|
packet += struct.pack("<4sBIQB", self.MAGIC, self.VERSION, self.seq, ts_ms, hand_count)
|
||||||
|
|
||||||
|
for hid in range(hand_count):
|
||||||
|
h = hands3d[hid]
|
||||||
|
pts = np.asarray(h["landmarks"], dtype=np.float32)
|
||||||
|
if pts.shape != (21, 3):
|
||||||
|
continue
|
||||||
|
|
||||||
|
point_count = 21
|
||||||
|
handedness_code = self._encode_handedness(str(h.get("handedness", "Unknown")))
|
||||||
|
confidence = float(h.get("confidence", 0.0))
|
||||||
|
|
||||||
|
# per-hand header: hand_id(B), point_count(B), handedness(B), confidence(f)
|
||||||
|
packet += struct.pack("<BBBf", hid, point_count, handedness_code, confidence)
|
||||||
|
|
||||||
|
# points: 21*3 float32
|
||||||
|
packet += pts.tobytes(order="C")
|
||||||
|
|
||||||
|
self.sock.sendto(packet, self.addr)
|
||||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,77 @@
|
|||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import numpy as np
|
||||||
|
from Tbd.helper import CameraObject
|
||||||
|
|
||||||
|
def _np_to_list(a: np.ndarray):
|
||||||
|
return None if a is None else a.tolist()
|
||||||
|
|
||||||
|
def save_calibration_json(camera_list, file_path: str, logger=None):
|
||||||
|
payload = {
|
||||||
|
"version": 1,
|
||||||
|
"cameras": []
|
||||||
|
}
|
||||||
|
|
||||||
|
for cam in camera_list:
|
||||||
|
payload["cameras"].append({
|
||||||
|
"index": int(cam.index),
|
||||||
|
"pxWidth": int(cam.pxWidth),
|
||||||
|
"pxHeight": int(cam.pxHeight),
|
||||||
|
|
||||||
|
"camera_matrix": _np_to_list(cam.camera_matrix),
|
||||||
|
"distortion_coefficients": _np_to_list(cam.distortion_coefficients),
|
||||||
|
|
||||||
|
"rotation_matrix_world_to_camera": _np_to_list(cam.rotation_matrix_world_to_camera),
|
||||||
|
"translation_vector_world_to_camera": _np_to_list(cam.translation_vector_world_to_camera),
|
||||||
|
|
||||||
|
"camera_projection_matrix": _np_to_list(cam.camera_projection_matrix),
|
||||||
|
})
|
||||||
|
|
||||||
|
Path(file_path).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with open(file_path, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(payload, f, indent=2)
|
||||||
|
|
||||||
|
if logger:
|
||||||
|
logger.log(f"[Calibration] Saved calibration to {file_path}")
|
||||||
|
|
||||||
|
def _list_to_np(x, shape=None):
|
||||||
|
if x is None:
|
||||||
|
return None
|
||||||
|
a = np.array(x, dtype=np.float64)
|
||||||
|
if shape is not None:
|
||||||
|
a = a.reshape(shape)
|
||||||
|
return a
|
||||||
|
|
||||||
|
def load_calibration_json(file_path: str, logger=None):
|
||||||
|
file_path = str(file_path)
|
||||||
|
if not Path(file_path).exists():
|
||||||
|
raise FileNotFoundError(file_path)
|
||||||
|
|
||||||
|
with open(file_path, "r", encoding="utf-8") as f:
|
||||||
|
payload = json.load(f)
|
||||||
|
|
||||||
|
cams = []
|
||||||
|
for c in payload.get("cameras", []):
|
||||||
|
cam = CameraObject(
|
||||||
|
capture=None,
|
||||||
|
index=int(c["index"]),
|
||||||
|
pxWidth=int(c.get("pxWidth", 1536)),
|
||||||
|
pxHeight=int(c.get("pxHeight", 2048)),
|
||||||
|
)
|
||||||
|
|
||||||
|
cam.camera_matrix = _list_to_np(c.get("camera_matrix"), shape=(3,3))
|
||||||
|
cam.distortion_coefficients = _list_to_np(c.get("distortion_coefficients")) # keep native shape
|
||||||
|
|
||||||
|
cam.rotation_matrix_world_to_camera = _list_to_np(c.get("rotation_matrix_world_to_camera"), shape=(3,3))
|
||||||
|
tv = _list_to_np(c.get("translation_vector_world_to_camera"))
|
||||||
|
if tv is not None:
|
||||||
|
cam.translation_vector_world_to_camera = tv.reshape(3,1)
|
||||||
|
|
||||||
|
cam.camera_projection_matrix = _list_to_np(c.get("camera_projection_matrix"), shape=(3,4))
|
||||||
|
cam.load_video_capture()
|
||||||
|
cams.append(cam)
|
||||||
|
|
||||||
|
if logger:
|
||||||
|
logger.log(f"[Calibration] Loaded calibration from {file_path} ({len(cams)} cameras)")
|
||||||
|
|
||||||
|
return cams
|
||||||
@@ -0,0 +1,260 @@
|
|||||||
|
import numpy as np
|
||||||
|
import cv2
|
||||||
|
from PySide6.QtCore import QObject, QTimer
|
||||||
|
import mediapipe as mp
|
||||||
|
from Tbd.helper import Hand2D
|
||||||
|
from collections import defaultdict
|
||||||
|
from Helpers.HandUdpSender import HandUdpSender
|
||||||
|
|
||||||
|
import struct
|
||||||
|
import time
|
||||||
|
|
||||||
|
class TrackingController(QObject):
|
||||||
|
def __init__(self, cameraList, P_mats, logger=None, tick_ms=30):
|
||||||
|
super().__init__()
|
||||||
|
|
||||||
|
self.cameraList = cameraList
|
||||||
|
self.P_mats = P_mats
|
||||||
|
self.logger = logger
|
||||||
|
|
||||||
|
self.udp = HandUdpSender(host="127.0.0.1", port=9000)
|
||||||
|
|
||||||
|
self.timer = QTimer(self)
|
||||||
|
self.timer.timeout.connect(self._tick)
|
||||||
|
self.tick_ms = tick_ms
|
||||||
|
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
# init mediapipe here (or inject it)
|
||||||
|
|
||||||
|
self.mp_hands = mp.solutions.hands
|
||||||
|
self.hands = self.mp_hands.Hands(
|
||||||
|
static_image_mode=False,
|
||||||
|
max_num_hands=4,
|
||||||
|
model_complexity=1,
|
||||||
|
min_detection_confidence=0.5,
|
||||||
|
min_tracking_confidence=0.5,
|
||||||
|
)
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
# Guard: need projections
|
||||||
|
self.logger.log("Amount of P_mats:" + str(len(self.P_mats)))
|
||||||
|
if not self.P_mats or len(self.P_mats) < 2:
|
||||||
|
if self.logger: self.logger.log("[Tracking] Missing P_mats (need >=2 cameras calibrated)")
|
||||||
|
return
|
||||||
|
|
||||||
|
if self.running:
|
||||||
|
return
|
||||||
|
self.running = True
|
||||||
|
self.timer.start(self.tick_ms)
|
||||||
|
if self.logger: self.logger.log("[Tracking] Started")
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
if not self.running:
|
||||||
|
return
|
||||||
|
self.running = False
|
||||||
|
self.timer.stop()
|
||||||
|
if self.logger: self.logger.log("[Tracking] Stopped")
|
||||||
|
|
||||||
|
def _tick(self):
|
||||||
|
hands2d = self._detectHands_all_cameras()
|
||||||
|
if not hands2d:
|
||||||
|
return
|
||||||
|
|
||||||
|
# Group hands across cameras (you already have _allocateDetectedHands logic)
|
||||||
|
hand_groups = self._allocateDetectedHands(hands2d)
|
||||||
|
|
||||||
|
# Triangulate (your existing triangulation approach)
|
||||||
|
hands3d = self._triangulateHands(hand_groups)
|
||||||
|
|
||||||
|
if len(hands3d) > 0:
|
||||||
|
self.udp.send_hands3d(hands3d)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"hands2d={len(hands2d)} groups={len(hand_groups)} hands3d={len(hands3d)}",
|
||||||
|
flush=True
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(hands2d) > 0:
|
||||||
|
print("cam_ids:", sorted(set(h.camera_id for h in hands2d)), flush=True)
|
||||||
|
|
||||||
|
def _detectHands_all_cameras(self):
|
||||||
|
out = []
|
||||||
|
for camera in self.cameraList:
|
||||||
|
ok, frameBGR = camera.capture.read()
|
||||||
|
if not ok or frameBGR is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
h, w = frameBGR.shape[:2]
|
||||||
|
frameRGB = cv2.cvtColor(frameBGR, cv2.COLOR_BGR2RGB)
|
||||||
|
frameRGB.flags.writeable = False
|
||||||
|
|
||||||
|
hand_tracking_result = self.hands.process(frameRGB)
|
||||||
|
if not hand_tracking_result.multi_hand_landmarks:
|
||||||
|
continue
|
||||||
|
|
||||||
|
handed = hand_tracking_result.multi_handedness
|
||||||
|
for hi, lm_list in enumerate(hand_tracking_result.multi_hand_landmarks):
|
||||||
|
landmarks_px = np.array(
|
||||||
|
[[lm.x * w, lm.y * h] for lm in lm_list.landmark],
|
||||||
|
dtype=np.float32
|
||||||
|
)
|
||||||
|
|
||||||
|
label, score = "Unknown", 0.0
|
||||||
|
if hi < len(handed) and handed[hi].classification:
|
||||||
|
label = handed[hi].classification[0].label
|
||||||
|
score = handed[hi].classification[0].score
|
||||||
|
|
||||||
|
out.append(Hand2D(
|
||||||
|
camera_id=camera.index,
|
||||||
|
handedness=label,
|
||||||
|
score=score,
|
||||||
|
landmarks_px=landmarks_px
|
||||||
|
))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _allocateDetectedHands(self, hands2d):
|
||||||
|
frames_by_camera = defaultdict(list)
|
||||||
|
for hand in hands2d:
|
||||||
|
frames_by_camera[hand.camera_id].append(hand)
|
||||||
|
|
||||||
|
used = set()
|
||||||
|
hand_groups = []
|
||||||
|
reproj_threshold = 50.0
|
||||||
|
|
||||||
|
camera_ids = sorted(frames_by_camera.keys())
|
||||||
|
|
||||||
|
for cam_a in camera_ids:
|
||||||
|
if cam_a not in self.P_mats:
|
||||||
|
continue
|
||||||
|
for hand_a in frames_by_camera[cam_a]:
|
||||||
|
if id(hand_a) in used:
|
||||||
|
continue
|
||||||
|
|
||||||
|
group = {cam_a: hand_a}
|
||||||
|
used.add(id(hand_a))
|
||||||
|
|
||||||
|
P_a = self.P_mats[cam_a]
|
||||||
|
|
||||||
|
for cam_b in camera_ids:
|
||||||
|
if cam_b == cam_a:
|
||||||
|
continue
|
||||||
|
if cam_b not in self.P_mats:
|
||||||
|
continue
|
||||||
|
|
||||||
|
P_b = self.P_mats[cam_b]
|
||||||
|
best_hand = None
|
||||||
|
best_err = np.inf
|
||||||
|
|
||||||
|
for hand_b in frames_by_camera[cam_b]:
|
||||||
|
if id(hand_b) in used:
|
||||||
|
continue
|
||||||
|
|
||||||
|
err = self.pair_reprojection_error(
|
||||||
|
hand_a, hand_b, P_a, P_b, key_idxs=[0, 9]
|
||||||
|
)
|
||||||
|
if err < best_err:
|
||||||
|
best_err = err
|
||||||
|
best_hand = hand_b
|
||||||
|
|
||||||
|
if best_hand is not None and best_err < reproj_threshold:
|
||||||
|
group[cam_b] = best_hand
|
||||||
|
used.add(id(best_hand))
|
||||||
|
|
||||||
|
hand_groups.append(group)
|
||||||
|
|
||||||
|
return hand_groups
|
||||||
|
|
||||||
|
def triangulate_hands_with_metadata(self, hand_groups):
|
||||||
|
hands3d = []
|
||||||
|
for group in hand_groups:
|
||||||
|
if len(group) < 2:
|
||||||
|
continue
|
||||||
|
|
||||||
|
handedness, confidence = self.choose_metadata_from_group(group)
|
||||||
|
|
||||||
|
cams = list(group.keys())
|
||||||
|
P1 = self.P_mats[cams[0]]
|
||||||
|
P2 = self.P_mats[cams[1]]
|
||||||
|
lm1 = group[cams[0]].landmarks_px
|
||||||
|
lm2 = group[cams[1]].landmarks_px
|
||||||
|
|
||||||
|
pts3d = []
|
||||||
|
for i in range(lm1.shape[0]):
|
||||||
|
X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
|
||||||
|
pts3d.append(X)
|
||||||
|
|
||||||
|
hands3d.append({
|
||||||
|
"handedness": handedness, # "Left"/"Right"/"Unknown"
|
||||||
|
"confidence": confidence, # 0..1
|
||||||
|
"landmarks": np.asarray(pts3d, dtype=np.float32) # (21,3)
|
||||||
|
})
|
||||||
|
return hands3d
|
||||||
|
|
||||||
|
# --- math helpers (copy from your existing code) ---
|
||||||
|
def triangulate_point(self, P1, P2, x1, x2):
|
||||||
|
A = np.zeros((4, 4), dtype=np.float32)
|
||||||
|
A[0] = x1[0] * P1[2] - P1[0]
|
||||||
|
A[1] = x1[1] * P1[2] - P1[1]
|
||||||
|
A[2] = x2[0] * P2[2] - P2[0]
|
||||||
|
A[3] = x2[1] * P2[2] - P2[1]
|
||||||
|
_, _, Vt = np.linalg.svd(A)
|
||||||
|
X_h = Vt[-1]
|
||||||
|
X_h /= X_h[3]
|
||||||
|
|
||||||
|
return X_h[:3]
|
||||||
|
|
||||||
|
def project_point(self, P, X):
|
||||||
|
X_h = np.array([X[0], X[1], X[2], 1.0], dtype=np.float32)
|
||||||
|
x = P @ X_h
|
||||||
|
return np.array([x[0] / x[2], x[1] / x[2]], dtype=np.float32)
|
||||||
|
|
||||||
|
def pair_reprojection_error(self, hand_a, hand_b, P_a, P_b, key_idxs):
|
||||||
|
errors = []
|
||||||
|
for idx in key_idxs:
|
||||||
|
x1 = hand_a.landmarks_px[idx]
|
||||||
|
x2 = hand_b.landmarks_px[idx]
|
||||||
|
X = self.triangulate_point(P_a, P_b, x1, x2)
|
||||||
|
x1_hat = self.project_point(P_a, X)
|
||||||
|
x2_hat = self.project_point(P_b, X)
|
||||||
|
errors.append(np.linalg.norm(x1_hat - x1))
|
||||||
|
errors.append(np.linalg.norm(x2_hat - x2))
|
||||||
|
return float(np.mean(errors))
|
||||||
|
|
||||||
|
def choose_metadata_from_group(group: dict):
|
||||||
|
# group: {cam_id: Hand2D, ...}
|
||||||
|
best = max(group.values(), key=lambda h: float(getattr(h, "score", 0.0)))
|
||||||
|
handedness = getattr(best, "handedness", "Unknown")
|
||||||
|
confidence = float(getattr(best, "score", 0.0))
|
||||||
|
return handedness, confidence
|
||||||
|
|
||||||
|
def _triangulateHands(self, hand_groups):
|
||||||
|
hands3d = []
|
||||||
|
|
||||||
|
for group in hand_groups:
|
||||||
|
cams = list(group.keys())
|
||||||
|
if len(cams) < 2:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Pick metadata from best 2D view
|
||||||
|
best = max(group.values(), key=lambda hh: float(getattr(hh, "score", 0.0)))
|
||||||
|
handedness = getattr(best, "handedness", "Unknown")
|
||||||
|
confidence = float(getattr(best, "score", 0.0))
|
||||||
|
|
||||||
|
P1 = self.P_mats[cams[0]]
|
||||||
|
P2 = self.P_mats[cams[1]]
|
||||||
|
lm1 = group[cams[0]].landmarks_px
|
||||||
|
lm2 = group[cams[1]].landmarks_px
|
||||||
|
|
||||||
|
pts3d = []
|
||||||
|
for i in range(lm1.shape[0]):
|
||||||
|
X = self.triangulate_point(P1, P2, lm1[i], lm2[i])
|
||||||
|
pts3d.append(X)
|
||||||
|
|
||||||
|
hands3d.append({
|
||||||
|
"landmarks": np.asarray(pts3d, dtype=np.float32),
|
||||||
|
"handedness": handedness,
|
||||||
|
"confidence": confidence,
|
||||||
|
})
|
||||||
|
|
||||||
|
return hands3d
|
||||||
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user