import cv2 import numpy as np import torch import torch.nn as nn import torch.nn.functional as F import math from collections import deque, Counter import pandas as pd # ← added for rebuilding labels # Modern MediaPipe Tasks API (no legacy solutions module) import mediapipe as mp from mediapipe.tasks import python from mediapipe.tasks.python import vision # PyTorch ≥ 2.6 checkpoint loading fix import numpy as np import numpy.core.multiarray import numpy.dtypes torch.serialization.add_safe_globals([ np.ndarray, np.dtype, np.dtypes.Int64DType, np.core.multiarray._reconstruct ]) # =============================== # MODEL DEFINITION # =============================== class PositionalEncoding(nn.Module): def __init__(self, d_model, max_len=128): super().__init__() pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) self.register_buffer('pe', pe.unsqueeze(0)) def forward(self, x): return x + self.pe[:, :x.size(1)] class TransformerASL(nn.Module): def __init__(self, input_dim, num_classes, d_model=256, nhead=8, num_layers=4): super().__init__() self.proj = nn.Linear(input_dim, d_model) self.norm_in = nn.LayerNorm(d_model) self.pos = PositionalEncoding(d_model, max_len=128) enc_layer = nn.TransformerEncoderLayer( d_model=d_model, nhead=nhead, dim_feedforward=d_model * 4, dropout=0.15, activation='gelu', batch_first=True, norm_first=True ) self.encoder = nn.TransformerEncoder(enc_layer, num_layers=num_layers) self.head = nn.Sequential( nn.LayerNorm(d_model), nn.Dropout(0.25), nn.Linear(d_model, num_classes) ) def forward(self, x, key_padding_mask=None): x = self.proj(x) x = self.norm_in(x) x = self.pos(x) x = self.encoder(x, src_key_padding_mask=key_padding_mask) x = x.mean(dim=1) return self.head(x) # =============================== # FEATURE EXTRACTION # =============================== def get_features_sequence(landmarks_seq, max_frames=100): if landmarks_seq is None or len(landmarks_seq) == 0: return None, None wrist = landmarks_seq[:, 0:1, :] landmarks_seq = landmarks_seq - wrist scale = np.linalg.norm(landmarks_seq[:, 9], axis=1, keepdims=True) scale = np.maximum(scale, 1e-6) landmarks_seq = landmarks_seq / scale[:, :, np.newaxis] landmarks_seq = np.nan_to_num(landmarks_seq, nan=0.0, posinf=0.0, neginf=0.0) landmarks_seq = np.clip(landmarks_seq, -10, 10) tips = [4, 8, 12, 16, 20] bases = [1, 5, 9, 13, 17] curls = [np.linalg.norm(landmarks_seq[:, t] - landmarks_seq[:, b], axis=1) for b, t in zip(bases, tips)] curl_features = np.stack(curls, axis=1) deltas = np.zeros_like(landmarks_seq) if len(landmarks_seq) > 1: deltas[1:] = landmarks_seq[1:] - landmarks_seq[:-1] pos_flat = landmarks_seq.reshape(len(landmarks_seq), -1) delta_flat = deltas.reshape(len(landmarks_seq), -1) seq = np.concatenate([pos_flat, delta_flat, curl_features], axis=1) T, F = seq.shape if T < max_frames: pad = np.zeros((max_frames - T, F), dtype=np.float32) seq_padded = np.concatenate([seq, pad], axis=0) else: seq_padded = seq[:max_frames] mask = np.zeros(max_frames, dtype=bool) mask[:min(T, max_frames)] = True return seq_padded.astype(np.float32), mask # =============================== # MANUAL DRAWING FUNCTION # =============================== HAND_CONNECTIONS = [ (0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (0, 9), (9, 10), (10, 11), (11, 12), (0, 13), (13, 14), (14, 15), (15, 16), (0, 17), (17, 18), (18, 19), (19, 20), (5, 9), (9, 13), (13, 17) ] def draw_hand_landmarks(image, landmarks_list): h, w = image.shape[:2] # Draw connections (blue lines) for start_idx, end_idx in HAND_CONNECTIONS: start = landmarks_list[start_idx] end = landmarks_list[end_idx] start_pt = (int(start.x * w), int(start.y * h)) end_pt = (int(end.x * w), int(end.y * h)) cv2.line(image, start_pt, end_pt, (255, 0, 0), 2) # Draw landmarks (green circles) for lm in landmarks_list: x = int(lm.x * w) y = int(lm.y * h) cv2.circle(image, (x, y), 5, (0, 255, 0), -1) # =============================== # MAIN PROGRAM # =============================== print("Loading trained model...") checkpoint = torch.load("best_asl_transformer.pth", map_location="cpu") model = TransformerASL( input_dim=checkpoint['input_dim'], num_classes=checkpoint['num_classes'], d_model=checkpoint['d_model'], nhead=checkpoint['nhead'], num_layers=checkpoint['num_layers'] ) model.load_state_dict(checkpoint['model']) model.eval() # ─── FIX: Rebuild real sign names from train.csv ───────────────────── print("\n" + "=" * 70) print("Rebuilding sign name mapping from train.csv...") try: # CHANGE THIS PATH to where your train.csv actually is train_df = pd.read_csv("asl_kaggle/train.csv") # ← most important line! # Get unique signs, sorted (same order LabelEncoder usually uses) real_signs = sorted(train_df['sign'].unique()) # Use real sign names instead of numbers label_encoder_classes = real_signs print("SUCCESS! Loaded real sign names") print("Number of classes:", len(real_signs)) print("First 15 signs:", real_signs[:15]) print("=" * 70 + "\n") except Exception as e: print("ERROR loading train.csv:", e) print("Falling back to numeric labels (you'll see numbers instead of words)") label_encoder_classes = checkpoint['label_encoder_classes'] print("First 15 (still numbers):", label_encoder_classes[:15]) print("=" * 70 + "\n") # MediaPipe Tasks setup BaseOptions = python.BaseOptions HandLandmarker = vision.HandLandmarker HandLandmarkerOptions = vision.HandLandmarkerOptions VisionRunningMode = vision.RunningMode MODEL_PATH = "hand_landmarker.task" # Make sure this file is in the folder options = HandLandmarkerOptions( base_options=BaseOptions(model_asset_path=MODEL_PATH), running_mode=VisionRunningMode.VIDEO, num_hands=1, min_hand_detection_confidence=0.5, min_hand_presence_confidence=0.5, min_tracking_confidence=0.5 ) landmarker = HandLandmarker.create_from_options(options) # Buffers MAX_FRAMES = 100 sequence_buffer = [] prediction_buffer = deque(maxlen=15) cap = cv2.VideoCapture(0) if not cap.isOpened(): print("Cannot open webcam") exit() print("\nASL Recognition running - Press ESC to quit") print("Controls: ESC = quit | SPACE = clear | H = toggle landmarks\n") show_landmarks = True frame_timestamp_ms = 0 while cap.isOpened(): success, image = cap.read() if not success: break image = cv2.flip(image, 1) h, w = image.shape[:2] mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=image) frame_timestamp_ms += 33 results = landmarker.detect_for_video(mp_image, frame_timestamp_ms) overlay = image.copy() cv2.rectangle(overlay, (10, 10), (520, 340), (0, 0, 0), -1) cv2.addWeighted(overlay, 0.65, image, 0.35, 0, image) if results.hand_landmarks: hand_landmarks_list = results.hand_landmarks[0] if show_landmarks: draw_hand_landmarks(image, hand_landmarks_list) current_frame = np.array( [[lm.x, lm.y, lm.z] for lm in hand_landmarks_list], dtype=np.float32 ) sequence_buffer.append(current_frame) if len(sequence_buffer) > MAX_FRAMES: sequence_buffer = sequence_buffer[-MAX_FRAMES:] if len(sequence_buffer) >= 10: seq_np = np.array(sequence_buffer) feats, mask = get_features_sequence(seq_np, MAX_FRAMES) if feats is not None: x = torch.from_numpy(feats).float().unsqueeze(0) key_padding_mask = torch.from_numpy(~mask).unsqueeze(0) with torch.no_grad(): logits = model(x, key_padding_mask=key_padding_mask) probs = F.softmax(logits, dim=-1)[0] pred_idx = torch.argmax(probs).item() conf = probs[pred_idx].item() # Now using real sign names! sign = label_encoder_classes[pred_idx] if conf > 0.40: prediction_buffer.append(sign) final_sign = sign final_conf = conf if len(prediction_buffer) >= 6: final_sign = Counter(prediction_buffer).most_common(1)[0][0] try: final_conf = probs[label_encoder_classes.index(final_sign)].item() except: pass color = (0, 255, 100) if final_conf > 0.75 else (0, 220, 220) cv2.putText(image, f"Sign: {final_sign}", (25, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.8, color, 4) cv2.putText(image, f"Conf: {final_conf:.1%}", (25, 110), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (220, 220, 220), 2) top3_p, top3_i = torch.topk(probs, 3) for i, (p, idx) in enumerate(zip(top3_p, top3_i)): s = label_encoder_classes[idx.item()] cv2.putText(image, f"{i + 1}. {s:<18} {p:.1%}", (25, 155 + i * 40), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (200, 200, 200), 2) else: if len(sequence_buffer) < 25: sequence_buffer.clear() cv2.putText(image, "No hand detected", (25, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 0, 255), 3) cv2.putText(image, "ESC:quit SPACE:clear H:landmarks", (w - 480, h - 25), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (180, 180, 180), 1) cv2.imshow("ASL Recognition", image) key = cv2.waitKey(1) & 0xFF if key == 27: break elif key == 32: sequence_buffer.clear() prediction_buffer.clear() print("Buffers cleared") elif key in (ord('h'), ord('H')): show_landmarks = not show_landmarks print(f"Landmarks display: {'ON' if show_landmarks else 'OFF'}") cap.release() cv2.destroyAllWindows() landmarker.close() print("Recognition stopped.")