poses 3d-model control mvp

Author: MathematicLove

models/characters/CesiumMan.glb

@@ -0,0 +1,236 @@
+"""Control a 3D character with body pose keypoints."""
+
+from __future__ import annotations
+
+import argparse
+import time
+
+import cv2
+import numpy as np
+
+import ui
+from camera import CameraStream
+from character import MODEL_FILES, Character
+from mesh3d import MeshRenderer
+from pose import PoseTracker
+
+WINDOW = "Hand Tracking Drawing"
+VIEWS = ("full3d", "pip_cam", "pip_3d")
+VIEW_LABELS = {
+    "full3d": "3D CHARACTER",
+    "pip_cam": "3D CHARACTER + camera inset",
+    "pip_3d": "CAMERA + character inset",
+}
+PIP_RATIO = 0.30
+MODEL_NAMES = list(MODEL_FILES.keys())
+
+HELP_LINES_POSE = (
+    "POSES",
+    "  Stand so your shoulders, hips, arms and legs are visible",
+    "  The character mirrors your pose in real time",
+    "",
+    "KEYS",
+    "  K cycle view (3D only / cam inset / 3D inset)",
+    "  M next model    N previous model",
+    "  Y / P yaw / pitch character view    R reset view",
+    "  D debug    H help    Q quit",
+)
+
+
+class AppPose:
+    """Body-pose driven 3D character viewer."""
+
+    def __init__(self, args: argparse.Namespace) -> None:
+        self.args = args
+        self.cam = CameraStream(args.camera, args.width, args.height)
+        self.h, self.w = self.cam.shape
+        print(f"[i] camera: {self.w}x{self.h} @ {self.cam.source_fps:.0f} FPS", flush=True)
+
+        self.tracker = PoseTracker(args.pose_model, imgsz=args.pose_size, conf=args.pose_conf)
+        print(f"[i] pose device: {self.tracker.device}", flush=True)
+
+        self.renderer = MeshRenderer(self.w, self.h)
+        self.model_index = 0
+        self.character: Character | None = None
+        self._load_character()
+
+        self.view_index = 0
+        self.yaw, self.pitch = 0.0, 0.0
+        self.show_help = False
+        self.show_debug = args.debug
+        self.toast_text = ""
+        self.toast_until = 0.0
+        self.fps = 0.0
+        self._last_t = time.perf_counter()
+        self._no_pose_since: float | None = None
+
+    def _load_character(self) -> None:
+        """(Re)load the active character model."""
+        name = MODEL_NAMES[self.model_index]
+        self.character = Character.load(name)
+        self.toast(f"Model: {name}")
+
+    def toast(self, text: str, seconds: float = 2.0) -> None:
+        """Show transient message."""
+        self.toast_text = text
+        self.toast_until = time.time() + seconds
+
+    def next_model(self, step: int) -> None:
+        """Switch to another character model."""
+        self.model_index = (self.model_index + step) % len(MODEL_NAMES)
+        self._load_character()
+
+    def reset_view(self) -> None:
+        """Back to the default camera angle."""
+        self.yaw, self.pitch = 0.0, 0.0
+
+    def render_character(self, canvas_shape: tuple[int, int], pose) -> np.ndarray:
+        """Render the character posed by one detected person, or the rest pose."""
+        h, w = canvas_shape
+        out = np.zeros((h, w, 3), dtype=np.uint8)
+        assert self.character is not None
+        ch = self.character
+
+        rots = ch.local_rotations(pose) if pose is not None else {}
+        globs = ch.pose_globals(rots)
+        parts = ch.skin(globs)
+
+        if pose is not None:
+            hip = ch.pose_hip_center(pose)
+            cx = float(hip[0]) if hip is not None else w * 0.5
+        else:
+            cx = w * 0.5
+        cy = h * 0.86
+        focal = 1.15 * h
+
+        return self.renderer.render(
+            out, parts, ch.triangles(),
+            origin=(cx, cy), focal=focal, distance=3.0,
+            yaw=self.yaw, pitch=self.pitch,
+        )
+
+    def compose(self, frame: np.ndarray, pose) -> np.ndarray:
+        """Build the output frame for the current view mode."""
+        view = VIEWS[self.view_index]
+        char_img = self.render_character((self.h, self.w), pose)
+
+        if view == "full3d":
+            out = char_img
+        elif view == "pip_cam":
+            out = char_img.copy()
+            self._blit_pip(out, frame)
+        else:
+            out = frame.copy()
+            self._blit_pip(out, char_img)
+        return out
+
+    def _blit_pip(self, out: np.ndarray, source: np.ndarray) -> None:
+        """Draw a picture-in-picture inset bottom-left."""
+        h, w = out.shape[:2]
+        pw, ph = int(w * PIP_RATIO), int(h * PIP_RATIO)
+        inset = cv2.resize(source, (pw, ph))
+        x0, y0 = 16, h - ph - 16
+        cv2.rectangle(out, (x0 - 3, y0 - 3), (x0 + pw + 3, y0 + ph + 3), (210, 210, 210), 2, cv2.LINE_AA)
+        out[y0:y0 + ph, x0:x0 + pw] = inset
+
+    def render(self, frame: np.ndarray, poses: list) -> np.ndarray:
+        """Compose the full HUD frame."""
+        pose = poses[0] if poses else None
+        out = self.compose(frame, pose)
+
+        h, w = out.shape[:2]
+        ui.panel(out, 0, 0, w, 60, 0.5)
+        ui.put_text(out, VIEW_LABELS[VIEWS[self.view_index]], (16, 10), 20, (255, 255, 255))
+        name = MODEL_NAMES[self.model_index]
+        status = f"{name}   {'tracking' if pose is not None else 'no person detected'}"
+        ui.put_text(out, status, (16, 38), 16,
+                    (140, 230, 140) if pose is not None else (200, 160, 100))
+
+        right = f"{self.fps:.0f} FPS   {self.tracker.device.upper()}"
+        rw = ui._render_text(right, 16, (200, 200, 200))[0].shape[1]
+        ui.put_text(out, right, (w - rw - 16, 14), 16, (200, 200, 200))
+        ui.put_text(out, "H help", (w - rw - 16, 38), 15, (140, 140, 140))
+
+        if self.toast_text and time.time() < self.toast_until:
+            tw = ui._render_text(self.toast_text, 18, (255, 255, 255))[0].shape[1]
+            ui.panel(out, w // 2 - tw // 2 - 14, 74, tw + 28, 36, 0.65)
+            ui.put_text(out, self.toast_text, (w // 2 - tw // 2, 82), 18, (255, 255, 255))
+
+        if self.show_debug and pose is not None:
+            ui.panel(out, 12, h - 120, 260, 100, 0.6)
+            ui.put_text(out, f"score {pose.confidence:.2f}", (20, h - 112), 15, (220, 220, 220))
+            ui.put_text(out, f"yaw {np.degrees(self.yaw):+.0f} pitch {np.degrees(self.pitch):+.0f}",
+                        (20, h - 90), 15, (220, 220, 220))
+        if self.show_help:
+            ui.draw_help(out, HELP_LINES_POSE)
+        return out
+
+    def handle_key(self, key: int) -> bool:
+        """Handle one keypress."""
+        if key in (ord("q"), 27):
+            return False
+        if key == ord("k"):
+            self.view_index = (self.view_index + 1) % len(VIEWS)
+            self.toast(VIEW_LABELS[VIEWS[self.view_index]])
+        elif key == ord("m"):
+            self.next_model(1)
+        elif key == ord("n"):
+            self.next_model(-1)
+        elif key == ord("y"):
+            self.yaw += 0.25
+        elif key == ord("p"):
+            self.pitch = float(np.clip(self.pitch + 0.2, -1.2, 1.2))
+        elif key == ord("r"):
+            self.reset_view()
+            self.toast("View reset")
+        elif key == ord("d"):
+            self.show_debug = not self.show_debug
+            self.toast("Debug " + ("on" if self.show_debug else "off"))
+        elif key == ord("h"):
+            self.show_help = not self.show_help
+        return True
+
+    def run(self) -> None:
+        """Run the main loop."""
+        cv2.namedWindow(WINDOW, cv2.WINDOW_NORMAL)
+        cv2.resizeWindow(WINDOW, self.w, self.h)
+        print("[i] window open. H help, Q quit.", flush=True)
+        self.toast("Stand back so your full body is visible")
+
+        reason = "loop finished"
+        frames = 0
+        seq = -1
+        while True:
+            frame, seq = self.cam.wait_next(seq)
+            if self.cam.failed:
+                reason = "camera stopped delivering frames"
+                break
+            if not self.args.no_mirror:
+                frame = cv2.flip(frame, 1)
+            if frame.shape[:2] != (self.h, self.w):
+                frame = cv2.resize(frame, (self.w, self.h))
+
+            now = time.perf_counter()
+            dt = max(now - self._last_t, 1e-4)
+            self._last_t = now
+            self.fps += 0.12 * (1.0 / dt - self.fps)
+
+            poses = self.tracker(frame, dt)
+            out = self.render(frame, poses)
+            cv2.imshow(WINDOW, out)
+
+            key = cv2.waitKey(1) & 0xFF
+            frames += 1
+            if frames % 150 == 0:
+                print(f"[i] frames: {frames}  {self.fps:.1f} FPS  people: {len(poses)}", flush=True)
+
+            if key != 255 and not self.handle_key(key):
+                reason = f"quit key pressed ({key})"
+                break
+            if cv2.getWindowProperty(WINDOW, cv2.WND_PROP_VISIBLE) < 1:
+                reason = "window closed"
+                break
+
+        print(f"[i] stopped: {reason} (frames processed: {frames})", flush=True)
+        self.cam.release()
+        cv2.destroyAllWindows()