From 4f0c54fe28210229740db6dcf1a786bc6c8cd67e Mon Sep 17 00:00:00 2001
From: charlie-rasberry <charlie.rasberry@outlook.com>
Date: Mon, 23 Feb 2026 16:26:48 +0000
Subject: [PATCH] Added training loop for the MTL architecture on the original
 distribution

---
 .gitignore                                    |   2 +
 ...events.out.tfevents.1771860635.mik.38652.0 | Bin 0 -> 88 bytes
 ...events.out.tfevents.1771860871.mik.34960.0 | Bin 0 -> 88 bytes
 ...events.out.tfevents.1771861405.mik.44772.0 | Bin 0 -> 88 bytes
 ...events.out.tfevents.1771861946.mik.10220.0 | Bin 0 -> 1893 bytes
 src/__pycache__/dataset.cpython-313.pyc       | Bin 2184 -> 2543 bytes
 src/dataset.py                                |   3 +-
 src/train.py                                  | 206 +++++++++++++++---
 8 files changed, 174 insertions(+), 37 deletions(-)
 create mode 100644 runs/fashion_trainer_20260223_153035/events.out.tfevents.1771860635.mik.38652.0
 create mode 100644 runs/fashion_trainer_20260223_153431/events.out.tfevents.1771860871.mik.34960.0
 create mode 100644 runs/fashion_trainer_20260223_154325/events.out.tfevents.1771861405.mik.44772.0
 create mode 100644 runs/fashion_trainer_20260223_155226/events.out.tfevents.1771861946.mik.10220.0

diff --git a/.gitignore b/.gitignore
index f57e31f..acb32c8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,5 @@ models/
 .ipynb_checkpoints/
 *.csv
 backup/*.csv
+runs/
+outputs/
\ No newline at end of file
diff --git a/runs/fashion_trainer_20260223_153035/events.out.tfevents.1771860635.mik.38652.0 b/runs/fashion_trainer_20260223_153035/events.out.tfevents.1771860635.mik.38652.0
new file mode 100644
index 0000000000000000000000000000000000000000..cefdbaf95f2f518e6dcdee5d4b2ca73c14b81a01
GIT binary patch
literal 88
zcmeZZfPjCKJmzxlJossuO!_THDc+=_#LPTB*Rs^S5-X!1JuaP+)V$*SqNM!9q7=R2
h(%js{qDsB;qRf)iBE3|Qs`#|boYZ)TNWU)sbpTN4Ad>(9

literal 0
HcmV?d00001

diff --git a/runs/fashion_trainer_20260223_153431/events.out.tfevents.1771860871.mik.34960.0 b/runs/fashion_trainer_20260223_153431/events.out.tfevents.1771860871.mik.34960.0
new file mode 100644
index 0000000000000000000000000000000000000000..d930585742770f4345d808a9c5449e933acdea43
GIT binary patch
literal 88
zcmeZZfPjCKJmzvPvcCOLCjFM96mL>dVrHJ6YguYuiIq{19+yr@YF=@EQBr<lQHox1
hX>M*}QKepaQD#YMkzOiDReV}zPHH?vWG!p0CIBs_AJ6~*

literal 0
HcmV?d00001

diff --git a/runs/fashion_trainer_20260223_154325/events.out.tfevents.1771861405.mik.44772.0 b/runs/fashion_trainer_20260223_154325/events.out.tfevents.1771861405.mik.44772.0
new file mode 100644
index 0000000000000000000000000000000000000000..73e50226aec54e7d47598ad4e7ac5c3603432dc4
GIT binary patch
literal 88
zcmeZZfPjCKJmzxV5e!e4O~2(R#hX-=n3<>NT9%quVr3Mh$E8z}npd1(l$4)Xl%iK$
hnwy(gRH;{9lv$Emq?Za(6`z)wlNt{ZdF1qXH303MAUyy8

literal 0
HcmV?d00001

diff --git a/runs/fashion_trainer_20260223_155226/events.out.tfevents.1771861946.mik.10220.0 b/runs/fashion_trainer_20260223_155226/events.out.tfevents.1771861946.mik.10220.0
new file mode 100644
index 0000000000000000000000000000000000000000..b53d518d2457727514689ccc48bf98a89959fee9
GIT binary patch
literal 1893
zcmaLTYe-XJ9LI5`BF|GRmmOy-O>-`CGSTvuTMudGvYB8lQn7W^qs?@yv&}5K5jlq!
zNG&YQR5B4MO_ZRUq(&D?$?8RkB^YRym&}YPx~%-adEmJ7;(2lSeV+$DDu>zccbTf1
z5{<V{xM%cl77J21MyHoY<*<6w#=v+v@-wk|qk&5?FkG74XwJ@NxFhnzoX*5@a`xX!
zo36`ZwX=Ul@*BK8;NCX_%L1wN{$?i++r7RnNQjWyU^M!h{yjkN`qijl^1^<ug7xFh
z^X`;mn-}`O7NVud>Hj*1$)fL>_bRK$&N%tN_T$~)X-YN;TQPD&!mR=Rwxu@3oT25|
zg9gq-SBJ>4+BMMX3sG}P=?o>I*UUb!8o47^h|*ccWad~~c*x8eO?2^YF;?E+6YB+~
zW8T;|sx=f!SF{`npk#~(*;Et#Dqf7Q{P^4@gTS(~dpqTv1c4{=Kq4a08f|R4Y}-K(
z9SFuB3hMStAdQfnilx-akgh-?<P2#xlgb&iOil(Jx>JfplZ*C}&?k;bc%k#IYZQ2X
zuz`Rs76(arprsQ73gM%9!36ZlNZ8H+EsfoRGk^RkC!sSSvIngh_rv?X*W?n=-9=FH
zK$ji&!&4VKnh5B^W(e#;Yd1yXevcSW0=oMKr2NoT$}qg(^wll`dZ4TV^FqIKtqfNb
z&>{jlF+sGM2YPUOo1*@L&p876Q9f+vfKHoH;V+>9^#rt_5h8og!kG=&vfJ$`0X^6Q
zB@eV!>4!rtg-;0R+mjI3g}xOOgWtzEIS|lmzd*_ltrscrrHN||1auMImc$EPIJ#Jo
znAuxPK);J*lX;+9qh2cnN%~p>nliw44(OK;wqa{TVjKaTS_+Xp=%UInj1Lv9BcMZC
zpyYv`8Qy?%N4c{E^mH!-cA+22qVYG$j>QD@{60wep)CU#PX%=DB%pg1I_L62UwTm*
zzEM0cfq?eblpNuKcKO<;cqYqwOF%c|!FCSl%DN~#)N)fpK!2}>$R2dll5jkty37#J
unH^B_K-YN(;!j88j07|uhQKa#g<g$2ueDnVX!j40@<Z1=$M{b9`l`QQN<&is

literal 0
HcmV?d00001

diff --git a/src/__pycache__/dataset.cpython-313.pyc b/src/__pycache__/dataset.cpython-313.pyc
index bf36b23c03d222b4d345d682f24bc94b31904039..01c9685b3bcc2e5c6c2df672836b078a182ff975 100644
GIT binary patch
delta 615
zcmeAWd@rp2nU|M~0SM-#&dIdlWMFs<;=lkKl<~P~qPnwK3`a1VRRjY=q7(x|47(CT
zFnbJJkkI4=77?jn4pT;mT2m$nUBs2nn9iulxiK%6i4SB0GZ24X29%vT`67$aWNzli
zO3`4OLF$9}KqM*%=CWj(JcBt<otqUXL8Mw9;??T&5~ntpPmfK7L6d*71Zy^@CgUyE
zl#<GV)XB}PwgSpPhZU)S2vs1V$uao?t1jCumYn>&^vQo&!<cv&Cx^1NbDl{(pLH_p
zqEzJM`)o2iS0ZAsM<iVYBKhRW|JbBCkE9;YI+%4)EO4?6J5V|*{(4l(MIcg0oovmX
z#i%=Z8GDG|Esps3+{Dbh`1mTtl*E!m{eq(W<kaHg)D->vqRjNnyu_UNk|Ll2z2xGu
zDrvXG<kY16{A@>0{feAi-J<-Y)S{9^-K50gR88(8jmiJm9TW_JZob8zR+OI`Ur>|^
z)|{GhiydToF_3FI*`7m!%Lb?j6t%@3lM^{)jlVExGID(gV`O0A>8QHQEP0bp`htx8
h0om)0ahDw9F2vVdcC5Y3SJ&wNfti6tsz?N=698BBje7t9

delta 266
zcmaDa+##s`nU|M~0SFqqr)M7EU|@I*;=lkal=0bVqPjCnI-@4r#&xkwe4IdGW+47-
z$Hc%eb#gw7(PVGt$CEEGN3wE(w18RMFqS9}h-JzMB!D28SC37FL6dLt0oLrvCTz9>
zN<hIPWe}kPBsAG4H?rwY-oh5f^p0UN4|_ZFWhwW`bJ%4RFUtqs5tE*tGbv|*$#V0B
z=F4ps+H7FHsBE$!;-a$MWik87FW6<6FDnF37U0NY)Sg_;5yGO$QKUXumeYaJcycYL
c1eZC`0+2I`<tNYQl;!%$pvlNpBmxuw0O~J2Gynhq

diff --git a/src/dataset.py b/src/dataset.py
index 379478e..3eaa170 100644
--- a/src/dataset.py
+++ b/src/dataset.py
@@ -51,7 +51,8 @@ class ReviewDataset(Dataset):
     
 if __name__ == "__main__":
     dataset = ReviewDataset("data/processed/original_train.csv", AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base"))
-    print(dataset.__getitem__(1))
+    # print(dataset.__getitem__(1))
+    
 
 
     
diff --git a/src/train.py b/src/train.py
index e7734d6..b28d18b 100644
--- a/src/train.py
+++ b/src/train.py
@@ -1,69 +1,203 @@
 # train.py
+# some code directly from pytorch docs https://docs.pytorch.org/tutorials/beginner/introyt/trainingyt.html
+from datetime import datetime
 import torch
+import random
 from sklearn.utils.class_weight import compute_class_weight
 import numpy as np
 import torch.nn as nn
 from torch.utils.data import DataLoader
 from transformers import AutoTokenizer
 import pandas as pd
+from torch.utils.tensorboard import SummaryWriter
+import torch.optim as optim
+from transformers import get_linear_schedule_with_warmup
+from sklearn.metrics import classification_report, f1_score
 
 from dataset import ReviewDataset
 from model import Model
 
+SEED = 4321
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+random.seed(SEED)
+
+EPOCHS = 5
+PATIENCE = 3
+
 # class weights, training loop and early stopping
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-
-tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
-
-train = "data/processed/original_train.csv"
-val = "data/processed/original_val.csv"
-train_dataset = ReviewDataset(train, tokenizer)
-val_dataset = ReviewDataset(val, tokenizer)
-train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
-val_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)
-
-model = Model().to(device)
-
-
-# move input_ids, attention_mask and labels to device in each batch
 
 # ------------------- Class weights -------------------
 # Using weights inversely proportional to class frequencies to avoid majority class bias, 
 # prioritize useful bug reports / feature requests
-def compute_weights(train_df, column):
-    classes = np.unique(train_df[column])
-    weights = compute_class_weight(class_weight='balanced', classes=classes, y=train_df[column])
+def compute_weights(df, column, device):
+    classes = np.unique(df[column])
+    weights = compute_class_weight(class_weight='balanced', classes=classes, y=df[column])
     return torch.tensor(weights, dtype=torch.float).to(device)
 
-# -------------------- Loss functions -------------------
-# just a later idea
-#    1.0 * bug_loss +
-#    1.0 * feature_loss +
-#    0.5 * aspect_loss +
-#    0.5 * sentiment_loss
+def main():
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    print("Using device:", device)
+    # Remove randomness
+    if torch.cuda.is_available():
+        print("GPU:", torch.cuda.get_device_name(0))
+        torch.cuda.manual_seed_all(SEED)
+        torch.cuda.manual_seed(SEED)
+
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
 
 
-# -------------------- Optimizer and scheduler -------------------
+    tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
+
+    train = "data/processed/original_train.csv"
+    val = "data/processed/original_val.csv"
+
+    train_dataset = ReviewDataset(train, tokenizer)
+    val_dataset = ReviewDataset(val, tokenizer)
+
+    training_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
+    validation_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)
+
+    model = Model().to(device)
+
+    train_df = pd.read_csv(train)
+    # move input_ids, attention_mask and labels to device in each batch
+    
+    # weights
+    bug_weights = compute_weights(train_df, 'bug_report', device)
+    feature_weights = compute_weights(train_df, 'feature_request', device)
+    aspect_weights = compute_weights(train_df, 'aspect', device)    
+    aspect_sentiment_weights = compute_weights(train_df, 'aspect_sentiment', device)
+
+    # Move tensors to cpu and conver to numpy for usage with sklearn classification report
+    # Use detatch() later for predictions
+    print("Bug report class weights:", bug_weights.cpu().numpy())
+    print("Feature request class weights:", feature_weights.cpu().numpy())
+    print("Aspect class weights:", aspect_weights.cpu().numpy())
+    print("Aspect sentiment class weights:", aspect_sentiment_weights.cpu().numpy())
+    
+    # -------------------- Loss Functions -------------------
+    #   for later
+    #   1.0 * bug_loss +
+    #   1.0 * feature_loss +
+    #   0.5 * aspect_loss +
+    #   0.5 * sentiment_loss
+
+    criterions = {
+        'bug_report': nn.CrossEntropyLoss(weight=bug_weights),
+        'feature_request': nn.CrossEntropyLoss(weight=feature_weights),
+        'aspect': nn.CrossEntropyLoss(weight=aspect_weights),
+        'aspect_sentiment': nn.CrossEntropyLoss(weight=aspect_sentiment_weights)
+    }
+
+    # -------------------- Optimizer and scheduler -------------------
+    optimizer = torch.optim.AdamW(
+        model.parameters(), 
+        lr=2e-5,        # change
+        weight_decay=0.01 
+        )
+    
+    total_steps = len(training_loader) * EPOCHS
+    warmup_steps = int(0.1 * total_steps) # 10% of steps for warmup
+    
+    scheduler = get_linear_schedule_with_warmup(
+        optimizer, 
+        num_warmup_steps=warmup_steps, 
+        num_training_steps=total_steps
+    )
+
+    # ------------------- Training loop -------------------
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    writer = SummaryWriter('runs/fashion_trainer_{}'.format(timestamp))
+
+    best_f1 = 0.0
+    patience_counter = 0
+    epoch_number = 0
 
 
 
+    # Initialize with inf to capture best validation loss easily
+    best_vloss = float('inf')
 
-# ------------------- Training loop -------------------
-# For each epoch:
+    for epoch in range(EPOCHS):
+        print(f"EPOCH {epoch_number + 1}")
+        model.train(True)
 
+        for step, batch in enumerate(training_loader):
+            optimizer.zero_grad()
 
+            # forward pass get logits for each head
+            input_ids = batch["input_ids"].to(device)
+            attention_mask = batch["attention_mask"].to(device)
 
+            outputs = model(input_ids, attention_mask)
+            # compute total loss
+            loss = 0
+            for task in criterions.keys():
+                labels = batch[task].to(device)
+                loss += criterions[task](outputs[task], labels)
 
-# ------------------- Stopping logic -------------------
-# After each epoch, find mean of 4 macro f1 scores
-# If there is no improvement for 3 epochs consecutively, stop training 
-# Prevents overfitting which saves time and resources
+            total_train_loss = loss.item()
 
+            loss.backward()
+            
+            # clip gradients to prevent exploding gradients
+            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
+            optimizer.step()
+            scheduler.step() 
+            if step % 50 == 0:
+                print(f" Batch {step}/{len(training_loader)} - Loss: {loss.item():.4f}")
 
+        avg_train_loss = total_train_loss / len(training_loader)
+        writer.add_scalar("Loss/train", avg_train_loss, epoch_number)
+        print(f"Average training loss: {avg_train_loss:.4f}")
 
+        # switch to evaluation mode
+        model.eval()
 
-train_df = pd.read_csv(train)
-bug_weights = compute_weights(train_df, 'bug_report')
-feature_weights = compute_weights(train_df, 'feature_request')
-aspect_weights = compute_weights(train_df, 'aspect')    
-aspect_sentiment_weights = compute_weights(train_df, 'aspect_sentiment')
\ No newline at end of file
+        all_preds = {task: [] for task in criterions.keys()}
+        all_labels = {task: [] for task in criterions.keys()}
+
+        with torch.no_grad():
+            for batch in validation_loader:
+                input_ids = batch["input_ids"].to(device)
+                attention_mask = batch["attention_mask"].to(device)
+
+                outputs = model(input_ids, attention_mask)
+                v_loss = 0.0
+                for task in criterions.keys():
+                    labels = batch[task].to(device)
+                    v_loss += criterions[task](outputs[task], labels).item() # detatch .item(*)
+                    preds = torch.argmax(outputs[task], dim=1).cpu().numpy()
+                    all_preds[task].extend(preds)
+                    all_labels[task].extend(labels.cpu().numpy())
+        avg_vloss = v_loss / len(validation_loader)
+        writer.add_scalar("Loss/val", avg_vloss, epoch_number)
+        print("\nValidation Metrics:")    
+        epoch_f1 = []
+        for task in criterions.keys():
+            task_f1 = f1_score(all_labels[task], all_preds[task], average='macro')
+            epoch_f1.append(task_f1)
+            writer.add_scalar(f"F1/val_{task}", task_f1, epoch_number)
+            print(f" {task} Macro F1: {task_f1:.4f}")
+        avg_macro_f1 = np.mean(epoch_f1)
+        writer.add_scalar("F1/val_macro_avg", avg_macro_f1, epoch_number)
+        print(f" Average Macro F1: {avg_macro_f1:.4f}")
+
+        if avg_macro_f1 > best_f1:
+            best_f1 = avg_macro_f1
+            patience_counter = 0
+            torch.save(model.state_dict(), f"outputs/best_mode.pt")
+            print(" New best model saved.")
+        else:
+            patience_counter += 1
+            print(f" No improvement. Patience counter: {patience_counter}/{PATIENCE}")
+            if patience_counter >= PATIENCE:
+                print(" Early stopping triggered.")
+                break
+    writer.close()
+    print("Training complete.")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file