HandTracking.py

import cv2
import mediapipe as mp
import numpy as np
from math import hypot
from ctypes import cast, POINTER
from comtypes import CLSCTX_ALL
from pycaw.pycaw import AudioUtilities, IAudioEndpointVolume
import pyautogui
import time
import os
import urllib.request

# Optimize PyAutoGUI for fast response times
pyautogui.FAILSAFE = False
pyautogui.PAUSE = 0

# Define the local model path and download URL for the modern Tasks API
MODEL_PATH = "hand_landmarker.task"
MODEL_URL = "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"

# Automatically download the pre-trained hand landmarker model if it doesn't exist locally
if not os.path.exists(MODEL_PATH):
    print("Notice: 'hand_landmarker.task' not found locally.")
    print("Downloading the official Google MediaPipe hand model...")
    try:
        req = urllib.request.Request(MODEL_URL, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req) as response, open(MODEL_PATH, 'wb') as out_file:
            out_file.write(response.read())
        print("Download complete! Model saved.")
    except Exception as e:
        print(f"Error downloading the model programmatically: {e}")
        print(f"Please download the file manually from: {MODEL_URL}")
        print(f"And place it inside the same folder as this script, renamed to: {MODEL_PATH}")
        exit()

# Try importing screen brightness control library
try:
    import screen_brightness_control as sbc
    HAS_SBC = True
except ImportError:
    HAS_SBC = False
    print("Notice: 'screen-brightness-control' library not found. Left-hand brightness control will be simulated.")

# Initialize webcam
video_capture = cv2.VideoCapture(0)
video_capture.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
video_capture.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)

# Set up Pycaw for Windows speaker control with fallback handling
try:
    devices = AudioUtilities.GetSpeakers()
    if hasattr(devices, '_dev'):
        interface = devices._dev.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
    else:
        interface = devices.Activate(IAudioEndpointVolume._iid_, CLSCTX_ALL, None)
    
    volume = cast(interface, POINTER(IAudioEndpointVolume))
    min_vol, max_vol, _ = volume.GetVolumeRange()
    print("Audio successfully initialized.")
except Exception as e:
    print(f"Audio initialization failed: {e}")
    volume = None

# Configure MediaPipe modern Tasks API
from mediapipe.tasks import python
from mediapipe.tasks.python import vision

base_options = python.BaseOptions(model_asset_path=MODEL_PATH)
options = vision.HandLandmarkerOptions(
    base_options=base_options,
    num_hands=2, 
    min_hand_detection_confidence=0.5,
    min_hand_presence_confidence=0.5,
    min_tracking_confidence=0.5
)
detector = vision.HandLandmarker.create_from_options(options)

# Hand skeleton connections map for custom drawing
HAND_CONNECTIONS = [
    (0, 1), (1, 2), (2, 3), (3, 4),               # Thumb
    (0, 5), (5, 6), (6, 7), (7, 8),               # Index
    (9, 10), (10, 11), (11, 12),                  # Middle
    (13, 14), (14, 15), (15, 16),                 # Ring
    (0, 17), (17, 18), (18, 19), (19, 20),         # Pinky
    (5, 9), (9, 13), (13, 17)                     # Knuckles/Palm
]

def draw_hand_skeleton(img, points):
    """Draws joint circles and skeleton lines manually."""
    for connection in HAND_CONNECTIONS:
        pt1 = points[connection[0]]
        pt2 = points[connection[1]]
        cv2.line(img, pt1, pt2, (200, 200, 200), 2)
    for pt in points:
        cv2.circle(img, pt, 5, (0, 0, 255), cv2.FILLED)

# Control mode toggle: "Media" or "Mouse"
control_mode = "Media"

# Signal smoothing settings (Exponential Moving Average)
vol_smoothed = -20.0
bright_smoothed = 50.0
alpha = 0.20 

# Mouse Control Settings & State
screen_w, screen_h = pyautogui.size()
mouse_x, mouse_y = screen_w // 2, screen_h // 2
prev_index_y = 0
click_triggered = False

# Mute Gesture State
pinch_duration = 0
mute_triggered = False
is_muted = False
prev_time = 0

if volume:
    try:
        is_muted = volume.GetMute()
    except Exception:
        is_muted = False

window_name = 'Interactive Hand Control HUD'

while True:
    ret, img = video_capture.read()
    if not ret:
        print("Error: Failed to read frame.")
        break

    # Flip the image horizontally for a natural mirror view
    img = cv2.flip(img, 1)
    h, w, _ = img.shape

    # Convert the frame to RGB
    frame_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    
    # Process the frame through the Tasks API detector
    mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=frame_rgb)
    detection_result = detector.detect(mp_image)

    # --- MOUSE MODE ACTIVE BOUNDARY BOX ---
    # Define an on-screen zone where hand coordinate movements are amplified to fit the full monitor screen
    box_x1, box_y1 = 250, 150
    box_x2, box_y2 = 1030, 570
    if control_mode == "Mouse":
        cv2.rectangle(img, (box_x1, box_y1), (box_x2, box_y2), (255, 0, 0), 2)
        cv2.putText(img, "ACTIVE TOUCHPAD ZONE", (box_x1 + 10, box_y1 - 10), 
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0), 1)

    # Check for hand landmarks in the results
    if detection_result.hand_landmarks:
        for hand_idx, hand_landmarks in enumerate(detection_result.hand_landmarks):
            
            # Map landmarks to pixel coordinates
            pts = []
            for lm in hand_landmarks:
                cx, cy = int(lm.x * w), int(lm.y * h)
                pts.append((cx, cy))
            
            # Draw skeleton overlay
            draw_hand_skeleton(img, pts)

            # Retrieve Hand classification (Right vs Left)
            hand_label = "Unknown"
            if len(detection_result.handedness) > hand_idx:
                hand_label = detection_result.handedness[hand_idx][0].category_name

            # Key fingers variables
            x1, y1 = pts[4][0], pts[4][1]   # Thumb tip
            x2, y2 = pts[8][0], pts[8][1]   # Index tip
            x3, y3 = pts[12][0], pts[12][1] # Middle tip
            cx, cy = (x1 + x2) // 2, (y1 + y2) // 2

            # Visual overlay on key fingers
            cv2.circle(img, (x1, y1), 10, (255, 255, 255), cv2.FILLED)
            cv2.circle(img, (x2, y2), 10, (255, 255, 255), cv2.FILLED)

            # --- MODE 1: MEDIA CONTROL MODE ---
            if control_mode == "Media":
                distance = hypot(x2 - x1, y2 - y1)
                gesture_range = [30, 250]

                # --- RIGHT HAND: VOLUME ---
                if hand_label == "Right" and volume:
                    target_vol = np.interp(distance, gesture_range, [min_vol, max_vol])
                    vol_smoothed = (alpha * target_vol) + ((1 - alpha) * vol_smoothed)
                    
                    try:
                        volume.SetMasterVolumeLevel(vol_smoothed, None)
                    except Exception:
                        pass

                    vol_percent = int(np.interp(vol_smoothed, [min_vol, max_vol], [0, 100]))
                    bar_y = int(np.interp(vol_smoothed, [min_vol, max_vol], [400, 150]))

                    cv2.rectangle(img, (1150, 150), (1180, 400), (80, 80, 80), 2)
                    cv2.rectangle(img, (1150, bar_y), (1180, 400), (255, 120, 0), cv2.FILLED)
                    cv2.putText(img, f"VOL: {vol_percent}%", (1100, 130), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 120, 0), 2)
                    cv2.putText(img, "Right Hand", (x1 - 30, y1 - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 120, 0), 2)

                    # Dynamic Mute / Unmute State Checking
                    if distance < 22:
                        pinch_duration += 1
                        # Trigger toggle once and lock until release
                        if pinch_duration > 15 and not mute_triggered:
                            is_muted = not is_muted
                            try:
                                volume.SetMute(1 if is_muted else 0, None)
                            except Exception:
                                pass
                            mute_triggered = True
                            pinch_duration = 0
                    else:
                        pinch_duration = 0
                        # Release state-lock only when hand is open (distance > 45)
                        if distance > 45:
                            mute_triggered = False
                            # Auto-unmute when volume starts being manually increased
                            if is_muted and distance > 55:
                                is_muted = False
                                try:
                                    volume.SetMute(0, None)
                                except Exception:
                                    pass

                # --- LEFT HAND: BRIGHTNESS ---
                elif hand_label == "Left":
                    target_bright = np.interp(distance, gesture_range, [0, 100])
                    bright_smoothed = (alpha * target_bright) + ((1 - alpha) * bright_smoothed)
                    bright_percent = int(bright_smoothed)

                    if HAS_SBC:
                        try:
                            sbc.set_brightness(bright_percent)
                        except Exception:
                            pass

                    bar_y = int(np.interp(bright_percent, [0, 100], [400, 150]))

                    cv2.rectangle(img, (100, 150), (130, 400), (80, 80, 80), 2)
                    cv2.rectangle(img, (100, bar_y), (130, 400), (0, 220, 255), cv2.FILLED)
                    cv2.putText(img, f"BRIGHT: {bright_percent}%", (50, 130), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 220, 255), 2)
                    cv2.putText(img, "Left Hand", (x1 - 30, y1 - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 220, 255), 2)

            # --- MODE 2: VIRTUAL TOUCHPAD & MOUSE MODE ---
            elif control_mode == "Mouse" and hand_label == "Right":
                # Determine which fingers are up
                # Comparing tip coordinate Y with corresponding joint knucle coordinate Y
                index_is_up = pts[8][1] < pts[6][1]
                middle_is_up = pts[12][1] < pts[10][1]

                # Movement Tracker: Only index finger up
                if index_is_up and not middle_is_up:
                    # Clip coordinates inside the boundary box
                    cx_clamped = np.clip(x2, box_x1, box_x2)
                    cy_clamped = np.clip(y2, box_y1, box_y2)

                    # Interpolate movement coordinates to fit whole computer monitor
                    target_screen_x = np.interp(cx_clamped, [box_x1, box_x2], [0, screen_w])
                    target_screen_y = np.interp(cy_clamped, [box_y1, box_y2], [0, screen_h])

                    # Smooth mouse coordinate displacement
                    mouse_x = (0.25 * target_screen_x) + (0.75 * mouse_x)
                    mouse_y = (0.25 * target_screen_y) + (0.75 * mouse_y)

                    pyautogui.moveTo(int(mouse_x), int(mouse_y))
                    cv2.circle(img, (x2, y2), 15, (0, 255, 0), cv2.FILLED)

                # Clicking and Scrolling: Both Index and Middle fingers up
                elif index_is_up and middle_is_up:
                    # Calculate distance between index and middle finger tips
                    finger_gap = hypot(x3 - x2, y3 - y2)

                    # Left Click: Fingers brought together (touching)
                    if finger_gap < 30:
                        cv2.line(img, (x2, y2), (x3, y3), (0, 255, 0), 4)
                        if not click_triggered:
                            pyautogui.click()
                            click_triggered = True
                    # Scroll Mode: Fingers spaced apart, moving vertically scrolls the page
                    else:
                        click_triggered = False
                        cv2.line(img, (x2, y2), (x3, y3), (255, 0, 0), 2)
                        
                        # Compare current index tip Y position with its previous position
                        if prev_index_y != 0:
                            y_displacement = y2 - prev_index_y
                            if abs(y_displacement) > 4:
                                # Negative displacement = Hand goes up -> Scroll Up
                                # Positive displacement = Hand goes down -> Scroll Down
                                pyautogui.scroll(-int(y_displacement * 2))
                    
                    prev_index_y = y2
                else:
                    prev_index_y = 0
                    click_triggered = False

    # --- GENERAL HUD OVERLAYS ---
    # Performance metric (FPS)
    curr_time = time.time()
    fps = 1 / (curr_time - prev_time) if (curr_time - prev_time) > 0 else 0
    prev_time = curr_time
    cv2.putText(img, f"FPS: {int(fps)}", (20, 45), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0), 2)

    # Mode HUD Display
    cv2.putText(img, f"ACTIVE MODE: {control_mode}", (w // 2 - 140, 45), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0, 255, 255), 2)

    # Semi-transparent Controls Guide panel
    overlay = img.copy()
    cv2.rectangle(overlay, (20, 80), (380, 200), (0, 0, 0), -1)
    img = cv2.addWeighted(overlay, 0.6, img, 0.4, 0)
    
    cv2.putText(img, "CONTROLS GUIDE:", (30, 105), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
    cv2.putText(img, "[Press 'M' to Toggle Modes]", (30, 125), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (0, 255, 255), 1)
    
    if control_mode == "Media":
        cv2.putText(img, "- R Hand Pinch: Adjust Volume", (30, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)
        cv2.putText(img, "- L Hand Pinch: Adjust Brightness", (30, 165), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)
        cv2.putText(img, "- Hold R Pinch (0%): Toggle Mute", (30, 185), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)
    else:
        cv2.putText(img, "- Raise Index: Move Mouse", (30, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)
        cv2.putText(img, "- Index + Middle Pinch: Left Click", (30, 165), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)
        cv2.putText(img, "- Index + Middle Apart: Move to Scroll", (30, 185), cv2.FONT_HERSHEY_SIMPLEX, 0.45, (255, 255, 255), 1)

    # Display Mute status
    if volume and is_muted:
        cv2.putText(img, "SYSTEM MUTED", (w // 2 - 120, h - 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 3)

    # Fallback status message if brightness library is missing
    if not HAS_SBC:
        cv2.putText(img, "Missing 'screen-brightness-control' library", (20, h - 20), 
                    cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1)

    cv2.imshow(window_name, img)
    
    # Check if the user closed the window using the window's standard 'X' button
    try:
        if cv2.getWindowProperty(window_name, cv2.WND_PROP_VISIBLE) < 1:
            break
    except cv2.error:
        pass
    
    # Check keyboard inputs
    key = cv2.waitKey(1) & 0xFF
    if key == ord('q'): # Exit the application
        break
    elif key == ord('m'): # Toggle Modes on 'M' keypress
        control_mode = "Mouse" if control_mode == "Media" else "Media"

video_capture.release()
cv2.destroyAllWindows()