Pycursor2.0.py

import cv2
import mediapipe as mp
import numpy as np
from pynput.mouse import Controller, Button
import screeninfo
from filterpy.kalman import KalmanFilter
import time
import os
import sys

def load_calibration_data():
    try:
        calibration_data = np.load("calibration.npy", allow_pickle=True).item()
        print("Loaded calibration data:", calibration_data)  # Print the calibration data
        # Check for required keys
        required_keys = ['top_left', 'top_right', 'bottom_left', 'bottom_right', 'threshold_slope', 'threshold_intercept']
        for key in required_keys:
            if key not in calibration_data:
                print(f"Error: Missing key '{key}' in calibration data.")
                exit()
        return (
            np.array(calibration_data["top_left"], dtype=np.float32),
            np.array(calibration_data["top_right"], dtype=np.float32),
            np.array(calibration_data["bottom_left"], dtype=np.float32),
            np.array(calibration_data["bottom_right"], dtype=np.float32),
            calibration_data["threshold_slope"],
            calibration_data["threshold_intercept"]
        )
    except FileNotFoundError:
        print("Calibration data not found. Please run calibrate_corners_and_pinch.py first.")
        exit()

def initialize_mouse_controller():
    return Controller()

def setup_mediapipe_hands():
    return mp.solutions.hands.Hands(max_num_hands=1, min_detection_confidence=0.8, model_complexity=0)

def open_webcam():
    cap = cv2.VideoCapture(0)
    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
    return cap

def get_screen_size():
    screen = screeninfo.get_monitors()[0]
    return screen.width, screen.height

def compute_perspective_transform(src_points, dst_points):
    return cv2.getPerspectiveTransform(src_points, dst_points)

def setup_kalman_filter():
    kf = KalmanFilter(dim_x=4, dim_z=2)
    kf.x = np.zeros(4)
    kf.P *= 1000.
    kf.F = np.array([[1, 0, 1, 0],
                     [0, 1, 0, 1],
                     [0, 0, 1, 0],
                     [0, 0, 0, 1]])
    kf.H = np.array([[1, 0, 0, 0],
                     [0, 1, 0, 0]])
    kf.R = np.eye(2) * 5
    kf.Q = np.eye(4) * 0.01
    return kf

def create_overlay(screen_width, screen_height):
    overlay = np.zeros((screen_height, screen_width, 4), dtype=np.uint8)
    cv2.namedWindow("Overlay", cv2.WND_PROP_FULLSCREEN)
    cv2.setWindowProperty("Overlay", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
    return overlay

def create_cursor_image(cursor_size):
    cursor_img = np.zeros((cursor_size, cursor_size, 4), dtype=np.uint8)
    cv2.circle(cursor_img, (cursor_size // 2, cursor_size // 2), cursor_size // 2, (0, 255, 255, 255), -1)
    cursor_img = cv2.GaussianBlur(cursor_img, (5, 5), 0)
    return cursor_img

def create_click_animation_image(click_anim_size):
    click_anim_img = np.zeros((click_anim_size, click_anim_size, 4), dtype=np.uint8)
    cv2.circle(click_anim_img, (click_anim_size // 2, click_anim_size // 2), click_anim_size // 2, (0, 255, 0, 255), 2)
    return click_anim_img

def main():
    # Load calibration data
    top_left, top_right, bottom_left, bottom_right, threshold_slope, threshold_intercept = load_calibration_data()

    # Initialize mouse controller
    mouse = initialize_mouse_controller()

    # Set up MediaPipe Hands
    hands = setup_mediapipe_hands()

    # Open webcam
    cap = open_webcam()

    # Get screen size
    screen_width, screen_height = get_screen_size()

    # Frame size for processing
    frame_width, frame_height = 320, 240

    # Define source and destination points for perspective transform
    src_points = np.array([top_left, top_right, bottom_right, bottom_left], dtype=np.float32)
    dst_points = np.array([[0, 0], [screen_width, 0], [screen_width, screen_height], [0, screen_height]], dtype=np.float32)

    # Compute perspective transform matrix
    perspective_matrix = compute_perspective_transform(src_points, dst_points)

    # Set up Kalman Filter for smoothing
    kf = setup_kalman_filter()

    # Pinch debouncing variables
    pinch_start_time = None
    debounce_delay = 0.1
    is_pinching = False

    # Click animation variables
    click_animation_start = None
    click_animation_duration = 0.3

    # Menu variables
    menu_active = False
    menu_center = (screen_width // 2, screen_height // 2)
    menu_radius = 100
    menu_items = ["Left Click", "Right Click", "Scroll"]
    menu_angles = [0, 120, 240]
    selected_item = None

    # Create a blank overlay window for rendering the cursor and animations
    overlay = create_overlay(screen_width, screen_height)

    # Create a custom cursor image
    cursor_size = 20
    cursor_img = create_cursor_image(cursor_size)

    # Create a click animation image
    click_anim_size = 40
    click_anim_img = create_click_animation_image(click_anim_size)

    # Move the system cursor off-screen
    mouse.position = (-100, -100)

    # Clear the console initially
    os.system("cls" if os.name == "nt" else "clear")

    while cap.isOpened():
        success, frame = cap.read()
        if not success:
            continue

        # Clear the overlay
        overlay.fill(0)

        # Flip and process frame
        frame = cv2.flip(frame, 1)
        small_frame = cv2.resize(frame, (frame_width, frame_height))
        small_frame_rgb = cv2.cvtColor(small_frame, cv2.COLOR_BGR2RGB)
        results = hands.process(small_frame_rgb)

        if results.multi_hand_landmarks:
            hand_landmarks = results.multi_hand_landmarks[0]

            # Get thumb and index fingertip positions
            thumb_tip = hand_landmarks.landmark[4]
            index_tip = hand_landmarks.landmark[8]
            thumb_x, thumb_y = int(thumb_tip.x * frame_width), int(thumb_tip.y * frame_height)
            index_x, index_y = int(index_tip.x * frame_width), int(index_tip.y * frame_height)

            # Calculate hand size (wrist to middle finger MCP)
            wrist = hand_landmarks.landmark[0]
            middle_mcp = hand_landmarks.landmark[9]
            hand_size = np.sqrt((wrist.x - middle_mcp.x) ** 2 + (wrist.y - middle_mcp.y) ** 2) * frame_width

            # Calculate midpoint for cursor
            midpoint_x = (thumb_x + index_x) / 2
            midpoint_y = (thumb_y + index_y) / 2

            # Apply perspective transform
            point = np.array([[midpoint_x, midpoint_y]], dtype=np.float32)
            point = np.array([point])
            transformed_point = cv2.perspectiveTransform(point, perspective_matrix)
            screen_x, screen_y = transformed_point[0][0]

            # Clamp to screen
            screen_x = max(0, min(screen_x, screen_width))
            screen_y = max(0, min(screen_y, screen_height))

            # Smooth with Kalman Filter
            kf.predict()
            kf.update(np.array([screen_x, screen_y]))
            smoothed_x, smoothed_y = kf.x[0], kf.x[1]

            # Move system cursor (but keep it off-screen)
            mouse.position = (-100, -100)

            # Draw the custom cursor on the overlay
            cursor_x, cursor_y = int(smoothed_x) - cursor_size // 2, int(smoothed_y) - cursor_size // 2
            x1, y1 = max(0, cursor_x), max(0, cursor_y)
            x2, y2 = min(screen_width, cursor_x + cursor_size), min(screen_height, cursor_y + cursor_size)
            if x2 > x1 and y2 > y1:
                overlay[y1:y2, x1:x2] = cursor_img[0:(y2-y1), 0:(x2-x1)]

            # Pinch detection with dynamic threshold
            distance = np.sqrt((thumb_x - index_x) ** 2 + (thumb_y - index_y) ** 2)
            pinch_threshold = threshold_slope * hand_size + threshold_intercept
            current_time = time.time()

            if distance < pinch_threshold:
                if pinch_start_time is None:
                    pinch_start_time = current_time
                elif current_time - pinch_start_time > debounce_delay and not is_pinching:
                    # Start click/drag
                    if not menu_active:
                        mouse.press(Button.left)
                        click_animation_start = current_time
                    is_pinching = True
            else:
                if is_pinching:
                    # End click/drag
                    if not menu_active:
                        mouse.release(Button.left)
                    is_pinching = False
                    pinch_start_time = None

            # Click animation
            if click_animation_start is not None:
                elapsed = current_time - click_animation_start
                if elapsed < click_animation_duration:
                    # Scale the animation based on elapsed time (shrinking effect)
                    scale = 1 - (elapsed / click_animation_duration)
                    anim_size = int(click_anim_size * (1 + scale))
                    anim_img = np.zeros((anim_size, anim_size, 4), dtype=np.uint8)
                    cv2.circle(anim_img, (anim_size // 2, anim_size // 2), int(anim_size // 2 * scale), (0, 255, 0, 255), 2)
                    anim_x, anim_y = int(smoothed_x) - anim_size // 2, int(smoothed_y) - anim_size // 2
                    ax1, ay1 = max(0, anim_x), max(0, anim_y)
                    ax2, ay2 = min(screen_width, anim_x + anim_size), min(screen_height, anim_y + anim_size)
                    if ax2 > ax1 and ay2 > ay1:
                        overlay[ay1:ay2, ax1:ax2] = anim_img[0:(ay2-ay1), 0:(ax2-ax1)]
                else:
                    click_animation_start = None

            # Gesture detection for menu (peace sign: index and middle fingers extended)
            index_tip = hand_landmarks.landmark[8]
            middle_tip = hand_landmarks.landmark[12]
            ring_tip = hand_landmarks.landmark[16]
            pinky_tip = hand_landmarks.landmark[20]
            index_mcp = hand_landmarks.landmark[5]
            middle_mcp = hand_landmarks.landmark[9]
            ring_mcp = hand_landmarks.landmark[13]
            pinky_mcp = hand_landmarks.landmark[17]

            # Check if index and middle fingers are extended, and ring and pinky are folded
            index_extended = (index_tip.y < index_mcp.y)
            middle_extended = (middle_tip.y < middle_mcp.y)
            ring_folded = (ring_tip.y > ring_mcp.y)
            pinky_folded = (pinky_tip.y > pinky_mcp.y)

            if index_extended and middle_extended and ring_folded and pinky_folded:
                menu_active = True
            else:
                menu_active = False

            # Draw the radial menu if active
            if menu_active:
                # Draw menu background (semi-transparent circle)
                temp_overlay = np.zeros_like(overlay)
                cv2.circle(temp_overlay, menu_center, menu_radius, (255, 255, 255, 128), -1)

                # Draw menu items
                selected_item = None
                for i, (item, angle) in enumerate(zip(menu_items, menu_angles)):
                    rad = np.deg2rad(angle)
                    item_x = int(menu_center[0] + menu_radius * 0.6 * np.cos(rad))
                    item_y = int(menu_center[1] - menu_radius * 0.6 * np.sin(rad))
                    item_dist = np.sqrt((smoothed_x - item_x) ** 2 + (smoothed_y - item_y) ** 2)
                    # Highlight if cursor is over the item
                    if item_dist < 30:
                        selected_item = item
                        cv2.circle(temp_overlay, (item_x, item_y), 25, (0, 255, 0, 255), -1)
                    else:
                        cv2.circle(temp_overlay, (item_x, item_y), 25, (0, 0, 255, 255), -1)
                    # Draw item text
                    cv2.putText(temp_overlay, item, (item_x - 40, item_y + 5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255, 255), 1)

                # Alpha blending for transparency
                alpha = temp_overlay[:, :, 3] / 255.0
                for c in range(3):
                    overlay[:, :, c] = (1 - alpha) * overlay[:, :, c] + alpha * temp_overlay[:, :, c]
                overlay[:, :, 3] = np.maximum(overlay[:, :, 3], temp_overlay[:, :, 3])

                # Perform action if pinching on a menu item
                if is_pinching and selected_item:
                    if selected_item == "Left Click":
                        mouse.click(Button.left)
                    elif selected_item == "Right Click":
                        mouse.click(Button.right)
                    elif selected_item == "Scroll":
                        mouse.scroll(0, -1)  # Example: scroll down
                    menu_active = False  # Close menu after selection

            # Update console output
            sys.stdout.write(f"\rDistance: {distance:.2f} Threshold: {pinch_threshold:.2f}")
            sys.stdout.flush()

        # Display the overlay
        cv2.imshow("Overlay", overlay)
        if cv2.waitKey(1) & 0xFF == 27:  # Exit on 'Esc'
            break

    cap.release()
    cv2.destroyAllWindows()
    # Clear the console on exit
    os.system("cls" if os.name == "nt" else "clear")

if __name__ == "__main__":
    main()