From cabf8aa9b5312e0e39b69253291917817a53eb95 Mon Sep 17 00:00:00 2001
From: charlie-rasberry <charlie.rasberry@outlook.com>
Date: Thu, 26 Feb 2026 20:39:19 +0000
Subject: [PATCH] Fixed evaluation indentation and other bugs

---
 .gitignore                            |   6 +-
 src/__pycache__/model.cpython-311.pyc | Bin 0 -> 4519 bytes
 src/evaluate.py                       | 211 +++++++++++++-------------
 3 files changed, 111 insertions(+), 106 deletions(-)
 create mode 100644 src/__pycache__/model.cpython-311.pyc

diff --git a/.gitignore b/.gitignore
index aaad02c..132f261 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,4 +10,8 @@ models/
 backup/*.csv
 runs/
 outputs/
-*.pt
\ No newline at end of file
+*.pt
+__pycache__/
+*.png
+*.jpg
+*.json
diff --git a/src/__pycache__/model.cpython-311.pyc b/src/__pycache__/model.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b609e0c40c9c2dce1b63424df7bc4a7d04613c07
GIT binary patch
literal 4519
zcmc&%U2GHC6~1H7_<tN5lK6K+2o2x>hAbqs)owy|fd~=uz>n0rL8FOhlGyQ2xHDmC
z)<}8SszyR;HcFMWiYiyCXb}65N3>7-R`tzBQ(=u1DN<Ecym`e|s(9)-Gqw}Q4JuE)
zGd^?gx#ym9?>*l==lah)?;=qCvQbL?%|poFvC$}8z4G>3hLCk45sAr=ER$femV_m1
zO;{P^Su$*vOK=RakZ~eezakPVb7@;c$ulb<FW_gEgx!$hK+5(WDTnNo?DG~JLpQ;O
zBjd`t6K=zv)0~4*cV#?TKEb1HnMr%=Mu_Bo1~YyEKeHq{px-Td<c?>c^M#=odlD|-
z^Ri3gzwsXE@j;Iten(_qtFG2o;wZ>Co`J3xrYY3g3A|4EsMHZ^@}TA0YaOuFC3ls&
zo1*(Ft$MGL`@t!gdm@ksf`yPAkb~)PL%dZdL!JgC%N_8}9TM}uL7|vc=f@V+{N4P#
zoJ%dqRPV&*t-K^>uIK43QJI&eiCcQd&n9l&&QHr!6^*{hWWxl08=ic75*oh-RLCU3
zNF-sA*dm!>B+FOMgcVVcfc7IM!GV2F@3@o7&1U4gV8hV)&)@$J%U?&6>^|jKNBwU-
z;JM#2>qKpOkdW5BDDCUn)6goz5Ve6|5J@*?4k&K*VSxzES*Y$kZF?9lSldR}8*(ip
z91U)Pq?<a^&IUJaoHc9}EUo^6IW#R77-+#+G>b%F2Q8EDBG<*FJe|+akKH)`D3d)$
zO&6V;78N<h>zuN<04{Y1LMoS11%cwy^^O^u&k74vR;ifE$&zlDb4hSG)h)T4Zof+N
z3;9J=w@FaJ?*s^mRJSGbxtY|g?w(6YlAIG1fQ!z~$)cn?Rq&yZ6SFe7l~v@-3`OYb
z?%ZNlNM=Mukrmy8`arQjMOD_Fg4rPm3QnpQ%JRFu&wn~~Pm!rI_4%AgGbwp$_`=}D
zDTOAdvWUp|!Y?WA!BX%5bmEN_c0jL!*!=f0*>Uhs>`(2l9l=+QV8s#E9O1Gfyc0O7
z1%}I)?^Xi$w7|V0`_!}39b8M7qLuEr)*T0id#AH|?Z$?*nbCR%wVqF3ep%_fsde6j
zE@w5=2On<{p0{9zB%26=2KwK?>bFo?Ck0Z1Z+ih>TE1agef`)9(*G5|1#7w)n;`0u
zUs)uJ`JJ)uiykm`FSb2qLEHYBH78METT%O%+mFfumtp`}*2(7<-I>aP`-POG=w4A(
z<(!(z=Y%YHn&Nz79L3N|{Xld-_*fMhz)-=#v<o{C1u+{9VEq`9ULd+1PynhG-DXyF
zTP8o7QWeDpy@-U~UK%k3XY%w5kxEN}gZ{2}pb=#X$TF!O>0Up(dUh-H-N?$>%8}dJ
zk=sRk)z`W1UUhGbYtf-?5uca-mt**B&uY<;if>f&jTUWHf7g0wHMB8Y@%L%|K4ADd
z*SsZLx#LvXaf(L4z;lM;Ieu=#(}YMD1l=VF0L#S;);)soU{TDNJqAc=KgjER9p_0o
zlMw{ag0WdaM_)14ND(sz!QE6QfB<wGPtC${Cu-y{;$gYdHS#`U%-v9rhoV^;(D<-`
z%^6|Z*oR~jlAsfVngRPiz`FiFA$mWE(-=^UvKOo$TT~1U^^)ET$?ajk8JJr}><iM;
zd&2YwjA;h*L&sQKLj)uMzOjNs$FOZ<_AWq}G`LBMOL>F-dxPig4SFutK{3;qxW(<G
zXvsX_N3qt41b2}7favymET>1X<(OWa6^umSnUO_xk;-PvEh-Cg65;~ph^~4=!9bD)
zVz6VZ85}T>nMREBfrx@AuxuU?BjkGx@HGQHo=N48K;X3e&w+loJG_?Bx?@`RP|^J_
z8wrNjmP%(Uf%976JSKvF1jC!o(wVIXmEfQj90V4>6Ybf&QJUW#tVD+(h!k1P5!#7-
zwE2_LFSd_YBA2wtCFBKn`cM7o=UZRC9INz?X#FF|>nj_taRgh^1a7c#UVaSc<%83N
zHI323D2Jy5BLVC`qm3b?EdhK`swkmn+De;F8>FA+DcmpKCTRyrgkf80lXMigf~{aL
zI3(*P3rJ=sX%CDtlM$=C5QJ1-$Ws+=ICU6O`N5*Bz#YZ}pYA&d!kFCz`Ek^A5y>Sa
zLr8{^Tm}+z9>`#{54%nPG0mFkOZPU+*4upDn%ShEfV_gy1(0nP6W;y6TH;N3*Ynz0
z6vaOehvhYYQ1cIL4H}QQ?Z;>9vgW^7wCzGJi>^jDW-9)E&EH?N)oi3YyncW6erdSU
zHK2736kXNON3TNzuR;S`!<Ep-TIl1Vw;GJT4xWA$JiXOj30}~G7m9o}a_n_v=v8EB
zd!`ckREvCCb_93BKdK%(SM5Gg?Y&TQay}O%CxbX$W~w;}lcuMT7;8YN))QSEnMg3T
zw<kIeH(DcykzOAMmflB9>!=<RbQUhQf)HbL{<i!uC4X^Mgd3`?#u#cuJ4SaQXCg1c
zt=EW$OUET(oL`{%qzt!J>3p81W>YyaBN&(Dcv5*tFMx7T>6#YR<eYK+-MiDoBDx18
za|$>x3l~_!4#mS_-nkuWo+jtwxttM1bsKma?hR)0H#X)p&fJcZfO*m}Z28PFi)t#P
z#Ic8dhWaov!;QQ7hYc=IxD1`m$&XZ>O~Ktu=Zrb(To!N523mCvI<kt+%<IfUom-%(
zoC^OB=EMd0IWad6{qoN735<cMjnI40ME5Db0a~*%3{!IvW|AqBTZg?>(p7GKYd$vv
zSCn?LGQF1i9{E4)b%u{K;N^Bw{OswC?~(t*Ugvd&u%6=5D{H814OQ9xa_d`V2g<E)
zl|55#eRqB5R(Q>Kt{lHv@l9yHiRBwrZ={$lM@PR4RlGMf@6F}$Djz5&HtbttWj<cv
z;~F1dzP{`66|b&z!ZY)AbS+x(^k|-*<!iehfAQLr@u%a<*Q(y5MS11g#u=^i<Q7+s
zk7%byE8Z)b_sa74t}n38ukv4ataa1~!-Wm<ge@|~!K#BVaz$~)zZR%CLXc_7##=i^
O*f1=!7Hx<d4*L&_*xfDw

literal 0
HcmV?d00001

diff --git a/src/evaluate.py b/src/evaluate.py
index 29f1ef8..6eb4d77 100644
--- a/src/evaluate.py
+++ b/src/evaluate.py
@@ -67,120 +67,121 @@ def main():
         model = SingleTaskModel(args.task, task_classes[args.task]).to(device)
         active_tasks = [args.task]
 
-        print(f"Loading weights from {args.model_path}...")
-        model.load_state_dict(torch.load(args.model_path, map_location=device))
-        model.eval()
+    print(f"Loading weights from {args.model_path}...")
+    model.load_state_dict(torch.load(args.model_path, map_location=device))
+    model.eval()
 
-        all_labels =      {task: [] for task in active_tasks}
-        all_preds =       {task: [] for task in active_tasks}
-        all_confidences = {task: [] for task in active_tasks}
+    all_labels =      {task: [] for task in active_tasks}
+    all_preds =       {task: [] for task in active_tasks}
+    all_confidences = {task: [] for task in active_tasks}
 
-        print("Running inference on test set").upper()
-        with torch.no_grad():
-            for batch in test_loader:
-                input_ids = batch['input_ids'].to(device)
-                attention_mask = batch['attention_mask'].to(device)
-                outputs = model(input_ids, attention_mask)
-                for task in active_tasks:
-                    labels = batch[task].to_device()
-                    logits = outputs[task]
-                    preds = torch.argmax(logits, dim=1)
+    print("Running inference on test set")
+    with torch.no_grad():
+        for batch in test_loader:
+            input_ids = batch['input_ids'].to(device)
+            attention_mask = batch['attention_mask'].to(device)
+            outputs = model(input_ids, attention_mask)
+            for task in active_tasks:
+                labels = batch[task].to(device)
+                logits = outputs[task]
+                preds = torch.argmax(logits, dim=1)
 
-                    probs = F.softmax(logits, dim=1)
-                    confidence = probs.max(dim=1).values
+                probs = F.softmax(logits, dim=1)
+                confidence = probs.max(dim=1).values
 
-                    all_labels[task].extend(labels.cpu().numpy())
-                    all_preds[task].extend(preds.cpu().numpy())
-                    all_confidences[task].extend(confidence.cpu().numpy())
-        
-        summary = {
-            "mode": args.mode,
-            "dataset": args.dataset,
-            "task": args.task,
-            "model_path": args.model_path,
-            "results": {}
+                all_labels[task].extend(labels.cpu().numpy())
+                all_preds[task].extend(preds.cpu().numpy())
+                all_confidences[task].extend(confidence.cpu().numpy())
+    
+    summary = {
+        "mode": args.mode,
+        "dataset": args.dataset,
+        "task": args.task,
+        "model_path": args.model_path,
+        "results": {}
+    }
+
+    test_df = pd.read_csv(test) # for later
+
+    for task in active_tasks:
+        print(f"\nFor Task: {task.upper()}\n")
+
+        labels_arr = np.array(all_labels[task])
+        preds_arr = np.array(all_preds[task])
+        conf_arr = np.array(all_confidences[task])
+
+        print(f"\nClassification Report")
+        report = classification_report(
+            labels_arr, 
+            preds_arr, 
+            target_names=label_names[task],
+            digits=4,
+            zero_division=0
+        )
+        print(report)
+
+        report_dict = classification_report(
+            labels_arr, 
+            preds_arr,
+            target_names=label_names[task],
+            output_dict=True,
+            zero_division=0
+        )
+
+        correct = (labels_arr == preds_arr)
+        mean_conf = conf_arr.mean()
+        mean_conf_correct = conf_arr[correct].mean() if correct.any() else 0
+        mean_conf_incorrect = conf_arr[~correct].mean() if (~correct).any() else 0
+
+        print(f"Overall Mean confidence: {mean_conf:.4f}")
+        print(f"Mean confidence for correct predictions: {mean_conf_correct:.4f}")
+        print(f"Incorrect Predictions confidence: {mean_conf_incorrect:.4f}")
+
+        # save summary to JSON
+        summary["results"][task] = {
+            "macro_f1": float(report_dict["macro avg"]["f1-score"]),
+            "macro_precision": float(report_dict["macro avg"]["precision"]),
+            "macro_recall": float(report_dict["macro avg"]["recall"]),
+            "confidence": {
+                "overall": float(mean_conf),
+                "correct": float(mean_conf_correct),
+                "incorrect": float(mean_conf_incorrect)
+            },
+            "per_class": report_dict
         }
 
-        test_df = pd.read_csv(test) # for later
+        # Confusion matrix
 
-        for task in active_tasks:
-            print(f"\nFor Task: {task.upper()}\n")
-
-            labels_arr = np.array(all_labels[task])
-            preds_arr = np.array(all_preds[task])
-            conf_arr = np.array(all_confidences[task])
-
-            print(f"\nClassification Report")
-            report = classification_report(
-                labels_arr, 
-                preds_arr, 
-                target_names=label_names[task],
-                digits=4,
-                zero_division=0
-            )
-            print(report)
-
-            report_dict = classification_report(
-                labels_arr, 
-                target_names=label_names[task],
-                output_dict=True,
-                zero_division=0
-            )
-
-            correct = (labels_arr == preds_arr)
-            mean_conf = conf_arr.mean()
-            mean_conf_correct = conf_arr[correct].mean() if correct.any() else 0
-            mean_conf_incorrect = conf_arr[~correct].mean() if (~correct).any() else 0
-
-            print(f"Overall Mean confidence: {mean_conf:.4f}")
-            print(f"Mean confidence for correct predictions: {mean_conf_correct:.4f}")
-            print(f"Incorrect Predictions confidence: {mean_conf_incorrect:.4f}")
-
-            # save summary to JSON
-            summary["results"][task] = {
-                "macro_f1": report_dict["macro avg"]["f1-score"],
-                "macro_precision": report_dict["macro avg"]["precision"],
-                "macro_recall": report_dict["macro avg"]["recall"],
-                "confidence": {
-                    "overall": mean_conf,
-                    "correct": mean_conf_correct,
-                    "incorrect": mean_conf_incorrect
-                },
-                "per_class": report_dict
-            }
-
-            # Confusion matrix
-
-            cm = confusion_matrix(labels_arr, preds_arr)
-            fig, ax = plt.subplots(figsize=(8, 6))
-            sns.heatmap(
-                cm, annot=True, fmt="d", cmap="Blues", cbar=False,
-                xticklabels=label_names[task], yticklabels=label_names[task],
-                ax=ax
-            )
-            ax.set_xlabel("Predicted Label", fontweight="bold")
-            ax.set_ylabel("True Label", fontweight="bold")
-            ax.set_title(f"{task.replace("_", " ").title()} Confusion Matrix ({args.mode.upper()})", fontweight="bold")
-            
-            run_name = args.task if args.mode == "stl" else "mtl"
-            cm_path = f"outputs/figures/cm_{args.mode}_{args.dataset}_{task}.png"
-            fig.savefig(cm_path, dpi = 150, bbox_inches='tight')
-            plt.close(fig)
-            print("Saved cm to path", cm_path)
-
-            test_df[f'{task}_pred'] = [label_names[task][p] for p in preds_arr] # Map to human readable
-            test_df[f'{task}_confidence'] = conf_arr
-
-        # to JSON
+        cm = confusion_matrix(labels_arr, preds_arr)
+        fig, ax = plt.subplots(figsize=(8, 6))
+        sns.heatmap(
+            cm, annot=True, fmt="d", cmap="Blues", cbar=False,
+            xticklabels=label_names[task], yticklabels=label_names[task],
+            ax=ax
+        )
+        ax.set_xlabel("Predicted Label", fontweight="bold")
+        ax.set_ylabel("True Label", fontweight="bold")
+        ax.set_title(f"{task.replace('_', ' ').title()} Confusion Matrix ({args.mode.upper()})", fontweight="bold")
+        
         run_name = args.task if args.mode == "stl" else "mtl"
-        json_path = f"outputs/eval_summary_{args.mode}_{run_name}_{args.dataset}.json"
-        with open(json_path, "w") as f:
-            json.dump(summary, f, indent=4)
-        print(f"Saved evaluation summary to {json_path}")
+        cm_path = f"outputs/figures/cm_{args.mode}_{args.dataset}_{task}.png"
+        fig.savefig(cm_path, dpi = 150, bbox_inches='tight')
+        plt.close(fig)
+        print("Saved cm to path", cm_path)
 
-        csv_path = f"outputs/test_predictions_{args.mode}_{run_name}_{args.dataset}.csv"
-        test_df.to_csv(csv_path, index=False)
-        print("Saved raw predictions to CSV at", csv_path)
+        test_df[f'{task}_pred'] = [label_names[task][p] for p in preds_arr] # Map to human readable
+        test_df[f'{task}_confidence'] = conf_arr
+
+    # to JSON
+    run_name = args.task if args.mode == "stl" else "mtl"
+    json_path = f"outputs/eval_summary_{args.mode}_{run_name}_{args.dataset}.json"
+    with open(json_path, "w") as f:
+        json.dump(summary, f, indent=4)
+    print(f"Saved evaluation summary to {json_path}")
+
+    csv_path = f"outputs/test_predictions_{args.mode}_{run_name}_{args.dataset}.csv"
+    test_df.to_csv(csv_path, index=False)
+    print("Saved raw predictions to CSV at", csv_path)
 
 if __name__ == "__main__":
     main()
\ No newline at end of file