From fa06e4d49e304757e1753f426ea4e0b8e6795263 Mon Sep 17 00:00:00 2001 From: Stupdi Go Date: Sat, 10 Jan 2026 23:41:56 -0600 Subject: [PATCH] chatgpt lock tf in pt 2 --- training.py | 350 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 201 insertions(+), 149 deletions(-) diff --git a/training.py b/training.py index b9f7021..f176e51 100644 --- a/training.py +++ b/training.py @@ -14,21 +14,13 @@ import torch.optim as optim from torch.utils.data import Dataset, DataLoader from sklearn.model_selection import train_test_split from sklearn.preprocessing import LabelEncoder -from collections import Counter from multiprocessing import Pool, cpu_count from functools import partial from tqdm import tqdm +from collections import Counter # =============================== -# DEVICE -# =============================== -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print(f"Using device: {device}") -if device.type == "cuda": - print("GPU:", torch.cuda.get_device_name(0)) - -# =============================== -# DATA LOADING / FEATURES +# DATA LOADING # =============================== def load_kaggle_asl_data(base_path): train_df = pd.read_csv(os.path.join(base_path, "train.csv")) @@ -36,116 +28,140 @@ def load_kaggle_asl_data(base_path): sign_to_idx = json.load(f) return train_df, sign_to_idx + def extract_hand_landmarks_from_parquet(path): try: df = pd.read_parquet(path) - left = df[df["type"]=="left_hand"] - right = df[df["type"]=="right_hand"] + left = df[df["type"] == "left_hand"] + right = df[df["type"] == "right_hand"] + hand = left if len(left) >= len(right) else right if len(hand) == 0: return None + frames = sorted(hand['frame'].unique()) landmarks_seq = [] - for f in frames: - lm_frame = hand[hand['frame']==f] + + for frame in frames: + lm_frame = hand[hand['frame'] == frame] lm_list = [] for i in range(21): - lm = lm_frame[lm_frame['landmark_index']==i] + lm = lm_frame[lm_frame['landmark_index'] == i] if len(lm) == 0: - lm_list.append([0.0,0.0,0.0]) + lm_list.append([0.0, 0.0, 0.0]) else: - lm_list.append([float(lm['x'].iloc[0]), float(lm['y'].iloc[0]), float(lm['z'].iloc[0])]) + lm_list.append([ + float(lm['x'].iloc[0]), + float(lm['y'].iloc[0]), + float(lm['z'].iloc[0]) + ]) landmarks_seq.append(lm_list) + return np.array(landmarks_seq, dtype=np.float32) except: return None -def get_features_sequence_augmented(landmarks_seq, max_frames=100): - if landmarks_seq is None or len(landmarks_seq)==0: + +def get_features_sequence(landmarks_seq, max_frames=100): + if landmarks_seq is None or len(landmarks_seq) == 0: return None, None + # Center on wrist - landmarks_seq -= landmarks_seq[:,0:1,:] - # Scale using wrist → middle finger MCP - scale = np.linalg.norm(landmarks_seq[:,0] - landmarks_seq[:,9], axis=1, keepdims=True) + landmarks_seq = landmarks_seq - landmarks_seq[:, 0:1, :] + + # Scale using wrist → middle finger MCP distance + scale = np.linalg.norm(landmarks_seq[:, 0] - landmarks_seq[:, 9], axis=1, keepdims=True) scale = np.maximum(scale, 1e-6) - landmarks_seq /= scale[:, np.newaxis, :] + landmarks_seq = landmarks_seq / scale[:, np.newaxis, :] + # Finger curl distances - tips = [4,8,12,16,20] - bases = [1,5,9,13,17] + tips = [4, 8, 12, 16, 20] + bases = [1, 5, 9, 13, 17] curl_features = [] - for b,t in zip(bases,tips): - curl_features.append(np.linalg.norm(landmarks_seq[:,t]-landmarks_seq[:,b], axis=1)) + for b, t in zip(bases, tips): + curl_features.append(np.linalg.norm(landmarks_seq[:, t] - landmarks_seq[:, b], axis=1)) curl_features = np.stack(curl_features, axis=1) # (T,5) + # Temporal deltas deltas = np.zeros_like(landmarks_seq) deltas[1:] = landmarks_seq[1:] - landmarks_seq[:-1] - # Flatten - seq = np.concatenate([landmarks_seq, deltas, curl_features], axis=1) - # Pad / truncate - T = seq.shape[0] + + # Flatten features along last axis + seq = np.concatenate([landmarks_seq, deltas, curl_features[:, :, np.newaxis]], axis=2) + seq = seq.reshape(seq.shape[0], -1) # (T, feature_dim) + + # Pad or truncate to max_frames + T, F = seq.shape if T < max_frames: - pad = np.zeros((max_frames - T, seq.shape[1]), dtype=np.float32) - seq = np.concatenate([seq, pad]) - else: - seq = seq[:max_frames] + pad = np.zeros((max_frames - T, F), dtype=np.float32) + seq = np.concatenate([seq, pad], axis=0) + elif T > max_frames: + seq = seq[:max_frames, :] + # Mask valid_mask = np.zeros(max_frames, dtype=bool) - valid_mask[:min(T,max_frames)] = True - return seq, valid_mask + valid_mask[:min(T, max_frames)] = True + + return seq.astype(np.float32), valid_mask + def process_row(row, base_path, max_frames=100): path = os.path.join(base_path, row["path"]) if not os.path.exists(path): return None, None, None - lm = extract_hand_landmarks_from_parquet(path) - if lm is None: + try: + lm = extract_hand_landmarks_from_parquet(path) + if lm is None: + return None, None, None + feat, mask = get_features_sequence(lm, max_frames) + if feat is None: + return None, None, None + return feat, mask, row["sign"] + except: return None, None, None - seq, mask = get_features_sequence_augmented(lm, max_frames) - if seq is None: - return None, None, None - return seq, mask, row["sign"] - -# =============================== -# DATASET -# =============================== -class ASLSequenceDataset(Dataset): - def __init__(self, X, masks, y): - self.X = torch.from_numpy(X).float() - self.masks = torch.from_numpy(masks) - self.y = torch.from_numpy(y).long() - def __len__(self): - return len(self.X) - def __getitem__(self, idx): - return self.X[idx], self.masks[idx], self.y[idx] # =============================== # TRANSFORMER MODEL # =============================== class PositionalEncoding(nn.Module): - def __init__(self,d_model,max_len=128): + def __init__(self, d_model, max_len=128): super().__init__() - pe = torch.zeros(max_len,d_model) - pos = torch.arange(0,max_len,dtype=torch.float).unsqueeze(1) - div_term = torch.exp(torch.arange(0,d_model,2).float()*(-math.log(10000.0)/d_model)) - pe[:,0::2] = torch.sin(pos*div_term) - pe[:,1::2] = torch.cos(pos*div_term) + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) self.register_buffer('pe', pe.unsqueeze(0)) - def forward(self,x): - return x + self.pe[:,:x.size(1)] + + def forward(self, x): + return x + self.pe[:, :x.size(1)] + class TransformerASL(nn.Module): - def __init__(self, input_dim=131, num_classes=250, d_model=192, nhead=6, num_layers=4): + def __init__(self, input_dim=63, num_classes=250, d_model=192, nhead=6, num_layers=4): super().__init__() - self.proj = nn.Linear(input_dim,d_model) + self.proj = nn.Linear(input_dim, d_model) self.norm_in = nn.LayerNorm(d_model) self.pos = PositionalEncoding(d_model) + enc_layer = nn.TransformerEncoderLayer( - d_model=d_model, nhead=nhead, dim_feedforward=d_model*4, - dropout=0.2, activation='gelu', batch_first=True, norm_first=True + d_model=d_model, + nhead=nhead, + dim_feedforward=d_model * 4, + dropout=0.15, + activation='gelu', + batch_first=True, + norm_first=True ) - self.encoder = nn.TransformerEncoder(enc_layer,num_layers=num_layers) - self.head = nn.Sequential(nn.LayerNorm(d_model), nn.Dropout(0.25), nn.Linear(d_model,num_classes)) - def forward(self,x,key_padding_mask=None): + self.encoder = nn.TransformerEncoder(enc_layer, num_layers=num_layers) + + self.head = nn.Sequential( + nn.LayerNorm(d_model), + nn.Dropout(0.25), + nn.Linear(d_model, num_classes) + ) + + def forward(self, x, key_padding_mask=None): x = self.proj(x) x = self.norm_in(x) x = self.pos(x) @@ -153,135 +169,171 @@ class TransformerASL(nn.Module): x = x.mean(dim=1) return self.head(x) -def create_padding_mask(lengths,max_len): - return torch.arange(max_len,device=lengths.device)[None,:] >= lengths[:,None] + +def create_padding_mask(lengths, max_len): + return torch.arange(max_len, device=lengths.device)[None, :] >= lengths[:, None] # =============================== -# MAIN +# MAIN TRAINING # =============================== def main(): - base_path = "asl_kaggle" # adjust path + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print(f"Using device: {device}") + if device.type == "cuda": + print("GPU:", torch.cuda.get_device_name(0)) + + base_path = "asl_kaggle" # ← set your dataset path max_frames = 100 MIN_SAMPLES_PER_CLASS = 6 - # Load metadata print("Loading metadata...") train_df, sign_to_idx = load_kaggle_asl_data(base_path) rows = [row for _, row in train_df.iterrows()] - # Extract features - print("Extracting features...") + print("Processing landmark sequences...") with Pool(cpu_count()) as pool: - results = list(tqdm(pool.imap(partial(process_row,base_path=base_path,max_frames=max_frames),rows), - total=len(rows))) + results = list(tqdm( + pool.imap( + partial(process_row, base_path=base_path, max_frames=max_frames), + rows + ), + total=len(rows), + desc="Extracting landmarks" + )) + X_list, masks_list, y_list = [], [], [] - for seq, mask, sign in results: - if seq is not None: - X_list.append(seq) + for feat, mask, sign in results: + if feat is not None and feat.shape[0] == max_frames: + X_list.append(feat) masks_list.append(mask) y_list.append(sign) + if not X_list: - print("No valid sequences found") + print("No valid sequences found. Check parquet files / paths.") return X = np.stack(X_list) masks = np.stack(masks_list) - print(f"{len(X)} sequences | shape: {X.shape}") + print(f"Loaded {len(X)} sequences | shape: {X.shape}") - # Normalize - mean = X.mean(axis=(0,1),keepdims=True) - std = X.std(axis=(0,1),keepdims=True)+1e-8 - X = (X - mean)/std + # Global normalization + X = np.clip(X, -5.0, 5.0) + mean = X.mean(axis=(0, 1), keepdims=True) + std = X.std(axis=(0, 1), keepdims=True) + 1e-8 + X = (X - mean) / std - # Labels le = LabelEncoder() y = le.fit_transform(y_list) - # Remove classes with too few samples counts = Counter(y) - valid_classes = [cls for cls,cnt in counts.items() if cnt>=MIN_SAMPLES_PER_CLASS] + valid_classes = [cls for cls, cnt in counts.items() if cnt >= MIN_SAMPLES_PER_CLASS] mask_valid = np.isin(y, valid_classes) - X = X[mask_valid]; masks = masks[mask_valid]; y = y[mask_valid] - le = LabelEncoder(); y = le.fit_transform(y) - print(f"{len(X)} samples | {len(le.classes_)} classes") + X = X[mask_valid] + masks = masks[mask_valid] + y = y[mask_valid] - # Split + le = LabelEncoder() + y = le.fit_transform(y) + print(f"{len(X)} samples | {len(le.classes_)} classes after filtering") + + # Train-test split X_train, X_test, masks_train, masks_test, y_train, y_test = train_test_split( X, masks, y, test_size=0.15, stratify=y, random_state=42 ) - # Datasets / loaders - train_dataset = ASLSequenceDataset(X_train, masks_train, y_train) - test_dataset = ASLSequenceDataset(X_test, masks_test, y_test) - train_loader = DataLoader(train_dataset, batch_size=64,shuffle=True,num_workers=4,pin_memory=True) - test_loader = DataLoader(test_dataset, batch_size=96,shuffle=False,num_workers=4,pin_memory=True) + # Dataset + class ASLSequenceDataset(Dataset): + def __init__(self, X, masks, y): + self.X = torch.from_numpy(X).float() + self.masks = torch.from_numpy(masks) + self.y = torch.from_numpy(y).long() - # Model - model = TransformerASL(input_dim=X.shape[2], num_classes=len(le.classes_)).to(device) - print(f"Model params: {sum(p.numel() for p in model.parameters()):,}") + def __len__(self): + return len(self.X) - # Class weights - counts = Counter(y_train) - class_weights = np.array([len(y_train)/ (len(counts)*counts[i]) for i in range(len(counts))],dtype=np.float32) - criterion = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(device),label_smoothing=0.05) - optimizer = optim.AdamW(model.parameters(), lr=3e-4, weight_decay=1e-4) + def __getitem__(self, idx): + return self.X[idx], self.masks[idx], self.y[idx] + + train_loader = DataLoader( + ASLSequenceDataset(X_train, masks_train, y_train), + batch_size=64, shuffle=True, num_workers=4, pin_memory=True + ) + test_loader = DataLoader( + ASLSequenceDataset(X_test, masks_test, y_test), + batch_size=96, shuffle=False, num_workers=4, pin_memory=True + ) + + model = TransformerASL( + input_dim=X.shape[2], + num_classes=len(le.classes_), + d_model=192, + nhead=6, + num_layers=4 + ).to(device) + print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}") + + criterion = nn.CrossEntropyLoss(label_smoothing=0.05) + optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-4) scheduler = optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10) - # Training / eval - def train_epoch(): + # Training + best_acc = 0.0 + patience = 15 + wait = 0 + epochs = 70 + + for epoch in range(epochs): model.train() - total_loss=0; correct=total=0 + total_loss = 0 + correct = total = 0 for x, mask, yb in tqdm(train_loader, desc="Train"): x, mask, yb = x.to(device), mask.to(device), yb.to(device) - lengths = mask.sum(dim=1) - pad_mask = create_padding_mask(lengths, x.size(1)) + key_mask = ~mask # True where padding optimizer.zero_grad(set_to_none=True) - logits = model(x,key_padding_mask=pad_mask) - loss = criterion(logits,yb) + logits = model(x, key_padding_mask=key_mask) + loss = criterion(logits, yb) loss.backward() - torch.nn.utils.clip_grad_norm_(model.parameters(),0.8) + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=0.8) optimizer.step() total_loss += loss.item() - correct += (logits.argmax(-1)==yb).sum().item() + correct += (logits.argmax(-1) == yb).sum().item() total += yb.size(0) - return total_loss/len(train_loader), correct/total*100 + train_acc = correct / total * 100 - @torch.no_grad() - def evaluate(): + # Eval model.eval() - correct=total=0 - for x, mask, yb in test_loader: - x, mask, yb = x.to(device), mask.to(device), yb.to(device) - lengths = mask.sum(dim=1) - pad_mask = create_padding_mask(lengths, x.size(1)) - logits = model(x,key_padding_mask=pad_mask) - correct += (logits.argmax(-1)==yb).sum().item() - total += yb.size(0) - return correct/total*100 if total>0 else 0.0 + correct = total = 0 + with torch.no_grad(): + for x, mask, yb in test_loader: + x, mask, yb = x.to(device), mask.to(device), yb.to(device) + key_mask = ~mask + logits = model(x, key_padding_mask=key_mask) + correct += (logits.argmax(-1) == yb).sum().item() + total += yb.size(0) + test_acc = correct / total * 100 + + print(f"[{epoch+1:2d}/{epochs}] Loss: {total_loss/len(train_loader):.4f} | " + f"Train: {train_acc:.2f}% | Test: {test_acc:.2f}%") - # Train loop - best_acc=0.0; patience=15; wait=0; epochs=70 - for epoch in range(epochs): - loss, train_acc = train_epoch() - test_acc = evaluate() - print(f"[{epoch+1:2d}/{epochs}] loss:{loss:.4f} train:{train_acc:.2f}% test:{test_acc:.2f}%") scheduler.step() - if test_acc>best_acc: - best_acc=test_acc; wait=0 + if test_acc > best_acc: + best_acc = test_acc + wait = 0 torch.save({ - 'model':model.state_dict(), - 'optimizer':optimizer.state_dict(), - 'label_encoder_classes':le.classes_, - 'acc':best_acc, - 'epoch':epoch - },"best_asl_transformer.pth") - print(" → New best saved") + 'model': model.state_dict(), + 'optimizer': optimizer.state_dict(), + 'label_encoder_classes': le.classes_, + 'acc': best_acc, + 'epoch': epoch + }, "best_asl_transformer.pth") + print(" → New best saved") else: - wait+=1 - if wait>=patience: + wait += 1 + if wait >= patience: print("Early stopping") break - print(f"\nBest test accuracy reached: {best_acc:.2f}%") -if __name__=="__main__": + print(f"\nBest test accuracy: {best_acc:.2f}%") + +if __name__ == "__main__": main()