From 99896c0873bf4620bf9c5202bde4c44b19b3831f Mon Sep 17 00:00:00 2001
From: charlie-rasberry <charlie.rasberry@outlook.com>
Date: Thu, 26 Feb 2026 20:15:19 +0000
Subject: [PATCH] Added evaluation pipeline

---
 src/__pycache__/dataset.cpython-311.pyc | Bin 0 -> 2613 bytes
 src/evaluate.py                         | 186 ++++++++++++++++++++++++
 src/model.py                            |   2 +-
 src/train.py                            |   6 +
 4 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 src/__pycache__/dataset.cpython-311.pyc

diff --git a/src/__pycache__/dataset.cpython-311.pyc b/src/__pycache__/dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e6d70e08aef5750ac5a60b77b6378b3ca7f6c0a7
GIT binary patch
literal 2613
zcmcguO>7fK6rT02*N$U5KZFz~Ar5f}a>-Atq5u^QL_tKAD%y~=1uJE>*_kAp*t=$S
z%}?Z@98i@?s6thw^pHaiX$m5T9654mZ>?mbMOY){)Dt(OaN^WAyV${SDLpXj{buIP
z+i%``GxNs3M<O8vWu2_+XX+68hj%JPe%;vn85(N{BP^s*%BzKx5Bk2en2}PFz-40E
zpOI6tfP82SVd)OSej;s&2t9<?E2#hpVtK*Gd$hxcNB{=_<$0LdD>%3%z!&g(Weap4
zy867I3SnUate8N_y60SQF)6UXdDT`eVnZ`BlC#Z8bAcH8BB2`BsFVuFUN=bA5J4#c
zBk)1SzB@un#Nr*4lDO3=KlTHEfJH8m>pHpdn*BKfn>49H2Q~MKzk8DaYsjwJjL>|w
zofm5T$s?s=3CjDRSB*_xgkF^&qY3mtxR~@aUly|<B`Q`l>pJt{8CGXkJY=DadQ(Xg
zW7eKaN=&jydWK0^)t;jfFp5S2EY8^l<yDXN`P8=-q1M#soJ!L=nHu`!)6b?Xs!d@p
zgag@IET|~Dq1%eG*jf{Sfr^~(7wag%GTLo!Sv~$ME;c8e=ERD0H&BXoLlrHx^+46I
z0?!^OBCQ$^=v3qOF?4F<mVYC1wQ-w3l<$$GkBQ(e4fD1@8=$f{cNc8}WC4W-157{o
zw&||eRc{1Eu)Lx9-ig(dKjrV{3&D1J0EAj~VW7h&a~JnVkgg#x3<O?zb8%x8A#BS1
z_7W`Y#+z(aExzRQ)g{M1EMjTfzvYKb4zE|1#Jq^*eYTD2rl@G}8V70v1m6VT_Zz!@
zpK-o&*#C>Kb^9~fZu;<e<^vVJ?NIH({ogayf3TUtA8e+``_6=;>oTOIhKo$5ZccI!
z_9UbRJ1Mzs7Rag?>&7e#*)(Tps;!#_i&A3esG-=zuuN*PI^D?TY(>Ww<-3zbRU3x5
zs*+Ky1s0ml%_@{+P0Ew0AKSOGgf+|%6<7!iT+0#5X0mE!iDt7#uU0HhkpW1`^f1uU
zP7ZMnM>uqINB|^*?zv@>o;J1p1jPat5R*m1{I*H8IVOAdGAV5uv)~{<woKIVO?rs8
zg2d2F?rSx(d56DVQD%v)+av=CAog|ws17v)hZTmTv5ZPh2UkB@?f6x5nz}bmIZb^l
zayg7z4y_IUGF)nGEw#tj&z27LKwXL_%JrzF`vsC)qAxkD1k1gst$X9R6B~FI8+sBO
z+PPMYopxfUOP#%iV<(-?Q_nijJ?T96xT)AV>U56ow#C=`HtfwCoB5ql=h&%79nP@}
zf0MuF3vE}5ZC9MOE4%TY_5A(w+hf~bJv!j@eNh<x+Ufh|>E$1s%hQFpR*Y*-T-$AJ
zUrleE*gUy;W=C`qCw4A7iQzxVAM=IQ@nY+^(>lJ}b#ya)e{%c#?Wsozr+>I`cHHT|
z^faY7DO~6x#V+D>k#Z!^7%C$V>OyV;gv*rj6P%R!C19u-q9`n+C>ayy(!3p3lxsOP
z?Tt_#NZQKbIKTtsa=1%Dc`k8*r%~$yz;g8|`$YfIGOC4;;O{7-D#%@aFz#LOrr_+{
z!i=i(Tz`aLUV~X`YQ(Y#9yF;ws~c)su_-7Fz^%F1enHj9v}rDkd^vbCo$06MG@-WI
zKMnsIlJ&Hg6UtfDz^X-gK4XpUP8`VDdfFP`{FEnc76wYgnlWhxZc)nfvgc|aA8eSR
zW=6@vnd9>k%mS}(RXWHO`4!_v=T-2G7tvul1TuboS-%04ML`hCK_o<$!~7|s1BJ@3
zgnA0KzY^*#)c(p+Qh=zv3-`ueaOunaq3;AFg;y4zh_QkgD@EH^GKG$_g^{a|4?81Q
d3rfB?vgC{`6{EME=<Ve({**-|wSlth#=mK!J<R|B

literal 0
HcmV?d00001

diff --git a/src/evaluate.py b/src/evaluate.py
index e69de29..29f1ef8 100644
--- a/src/evaluate.py
+++ b/src/evaluate.py
@@ -0,0 +1,186 @@
+# evauluate.py
+import os
+import torch
+import time
+import argparse
+import json
+import torch.nn.functional as F
+import argparse
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer
+from sklearn.metrics import classification_report, confusion_matrix, f1_score
+
+from dataset import ReviewDataset
+from model import Model, SingleTaskModel
+
+# TODO: load checkpoint, produce tables of evaluation figures
+SEED = 4321
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+# Label names for classification report, readable format instead of numeric
+label_names = {
+    'bug_report': ['No', 'Yes'],
+    'feature_request': ['No', 'Yes'],
+    'aspect': ['App', 'Driver', 'General', 'Payment', 'Pricing', 'Service'],
+    'aspect_sentiment': ['Positive', 'Neutral', 'Negative']
+}
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="RECLASS Evaluation Script")
+    parser.add_argument("--mode", type=str, required=True, choices=["mtl", "stl"], help="mtl or stl")
+    parser.add_argument("--task", type=str, default="all", choices=["all", "bug_report", "feature_request", "aspect", "aspect_sentiment"])
+    parser.add_argument("--dataset", type=str, required=True, choices=["original", "boosted"])
+    parser.add_argument("--model_path", type=str, required=True, help=".pt file path")
+    parser.add_argument("--batch_size", type=int, default=16)
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+    
+    print(f"Evaluating {args.mode.upper()} model on {args.dataset} dataset for task: {args.task}")
+
+    os.makedirs("outputs/figures", exist_ok=True)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+    test = f"data/processed/{args.dataset}_test.csv"
+    tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
+    test_dataset = ReviewDataset(test, tokenizer)
+    test_loader = DataLoader(test_dataset, batch_size=args.batch_size)
+    if args.mode == "mtl":
+        model = Model().to(device)
+        active_tasks = ['bug_report', 'feature_request', 'aspect', 'aspect_sentiment']
+    else: 
+        if args.task == "all":
+            raise ValueError("For STL, please specify a single task with --task")
+        task_classes = {
+            'bug_report': 2,
+            'feature_request': 2,
+            'aspect': 6,
+            'aspect_sentiment': 3
+            }
+        model = SingleTaskModel(args.task, task_classes[args.task]).to(device)
+        active_tasks = [args.task]
+
+        print(f"Loading weights from {args.model_path}...")
+        model.load_state_dict(torch.load(args.model_path, map_location=device))
+        model.eval()
+
+        all_labels =      {task: [] for task in active_tasks}
+        all_preds =       {task: [] for task in active_tasks}
+        all_confidences = {task: [] for task in active_tasks}
+
+        print("Running inference on test set").upper()
+        with torch.no_grad():
+            for batch in test_loader:
+                input_ids = batch['input_ids'].to(device)
+                attention_mask = batch['attention_mask'].to(device)
+                outputs = model(input_ids, attention_mask)
+                for task in active_tasks:
+                    labels = batch[task].to_device()
+                    logits = outputs[task]
+                    preds = torch.argmax(logits, dim=1)
+
+                    probs = F.softmax(logits, dim=1)
+                    confidence = probs.max(dim=1).values
+
+                    all_labels[task].extend(labels.cpu().numpy())
+                    all_preds[task].extend(preds.cpu().numpy())
+                    all_confidences[task].extend(confidence.cpu().numpy())
+        
+        summary = {
+            "mode": args.mode,
+            "dataset": args.dataset,
+            "task": args.task,
+            "model_path": args.model_path,
+            "results": {}
+        }
+
+        test_df = pd.read_csv(test) # for later
+
+        for task in active_tasks:
+            print(f"\nFor Task: {task.upper()}\n")
+
+            labels_arr = np.array(all_labels[task])
+            preds_arr = np.array(all_preds[task])
+            conf_arr = np.array(all_confidences[task])
+
+            print(f"\nClassification Report")
+            report = classification_report(
+                labels_arr, 
+                preds_arr, 
+                target_names=label_names[task],
+                digits=4,
+                zero_division=0
+            )
+            print(report)
+
+            report_dict = classification_report(
+                labels_arr, 
+                target_names=label_names[task],
+                output_dict=True,
+                zero_division=0
+            )
+
+            correct = (labels_arr == preds_arr)
+            mean_conf = conf_arr.mean()
+            mean_conf_correct = conf_arr[correct].mean() if correct.any() else 0
+            mean_conf_incorrect = conf_arr[~correct].mean() if (~correct).any() else 0
+
+            print(f"Overall Mean confidence: {mean_conf:.4f}")
+            print(f"Mean confidence for correct predictions: {mean_conf_correct:.4f}")
+            print(f"Incorrect Predictions confidence: {mean_conf_incorrect:.4f}")
+
+            # save summary to JSON
+            summary["results"][task] = {
+                "macro_f1": report_dict["macro avg"]["f1-score"],
+                "macro_precision": report_dict["macro avg"]["precision"],
+                "macro_recall": report_dict["macro avg"]["recall"],
+                "confidence": {
+                    "overall": mean_conf,
+                    "correct": mean_conf_correct,
+                    "incorrect": mean_conf_incorrect
+                },
+                "per_class": report_dict
+            }
+
+            # Confusion matrix
+
+            cm = confusion_matrix(labels_arr, preds_arr)
+            fig, ax = plt.subplots(figsize=(8, 6))
+            sns.heatmap(
+                cm, annot=True, fmt="d", cmap="Blues", cbar=False,
+                xticklabels=label_names[task], yticklabels=label_names[task],
+                ax=ax
+            )
+            ax.set_xlabel("Predicted Label", fontweight="bold")
+            ax.set_ylabel("True Label", fontweight="bold")
+            ax.set_title(f"{task.replace("_", " ").title()} Confusion Matrix ({args.mode.upper()})", fontweight="bold")
+            
+            run_name = args.task if args.mode == "stl" else "mtl"
+            cm_path = f"outputs/figures/cm_{args.mode}_{args.dataset}_{task}.png"
+            fig.savefig(cm_path, dpi = 150, bbox_inches='tight')
+            plt.close(fig)
+            print("Saved cm to path", cm_path)
+
+            test_df[f'{task}_pred'] = [label_names[task][p] for p in preds_arr] # Map to human readable
+            test_df[f'{task}_confidence'] = conf_arr
+
+        # to JSON
+        run_name = args.task if args.mode == "stl" else "mtl"
+        json_path = f"outputs/eval_summary_{args.mode}_{run_name}_{args.dataset}.json"
+        with open(json_path, "w") as f:
+            json.dump(summary, f, indent=4)
+        print(f"Saved evaluation summary to {json_path}")
+
+        csv_path = f"outputs/test_predictions_{args.mode}_{run_name}_{args.dataset}.csv"
+        test_df.to_csv(csv_path, index=False)
+        print("Saved raw predictions to CSV at", csv_path)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/src/model.py b/src/model.py
index ba796c1..f74b849 100644
--- a/src/model.py
+++ b/src/model.py
@@ -11,7 +11,7 @@ import torch.nn as nn
 # Each nn.linear is used to map RoBERTa's hidden representation onto the output space of each task head
 # Each hidden representation is size 768
 
-class SingleTaskModel(nn.Module): #   SINGLE TASK MODEL ARCHITECTURE
+class SingleTaskModel(nn.Module): #   TASK-SPECIFIC/SINGLE-TASK MODEL ARCHITECTURE
     def __init__(self, task_name, num_classes, dropout_rate=0.2):
         super().__init__()
         self.encoder = XLMRobertaModel.from_pretrained("FacebookAI/xlm-roberta-base")
diff --git a/src/train.py b/src/train.py
index 69a507f..e9d6f45 100644
--- a/src/train.py
+++ b/src/train.py
@@ -3,6 +3,7 @@
 import argparse # argparse for later switching to boosted data
 import os
 from datetime import datetime
+import time
 import torch
 import random
 import numpy as np
@@ -153,6 +154,7 @@ def main():
     )
 
     # ------------------- Training loop -------------------
+    start_time = time.time()
     timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
     writer = SummaryWriter(f'runs/reclass_{run_name}_{timestamp}')
 
@@ -256,6 +258,10 @@ def main():
 
     writer.close()
     print("Training complete.")
+    end_time = time.time()
+    print(f"Total training time: {end_time - start_time:.2f} seconds")
+    if torch.cuda.is_available():
+        print(f"Peak GPU memory usage: {torch.cuda.max_memory_allocated(device) / (1024**3)} GB")
 
 if __name__ == "__main__":
     main()
\ No newline at end of file