mik/ReClass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added: clearer input and output paths

Browse Source
This commit is contained in:
charlie-rasberry
2026-07-17 20:02:59 +01:00
parent 9eaa1a3bc2
commit 7ba50ac23d
5 changed files with 36 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/infer.py
View File

@@ -35,7 +35,7 @@ np.random.seed(SEED)
def parse_args():
parser = argparse.ArgumentParser(description="RECLASS, Multitask learning for review classification.")
parser.add_argument("--model_path", type=str, required=True, help=".pt file in outputs/")
parser.add_argument("--model", type=str, required=True, help=".pt file - just the filename not path")
parser.add_argument("--task", type=str, default="all", choices=["all", "bug_report", "feature_request", "aspect", "aspect_sentiment"])
parser.add_argument("--interactive", action="store_true", help="Loops reading input until exit()")
parser.add_argument("--text", action="store_true", help="Use command line text for input")
@@ -63,7 +63,7 @@ def main():
else:
print(f'{" "*15, "No GPUs available"}')
print(f'{"="*50}\n')
print(f"Running inference on: outputs/{args.model_path} using data/processed/{args.dataset}.csv")
print(f"Running inference on: outputs/{args.model} using data/processed/{args.dataset}.csv")
print("Loading model, tokenizer and datasets ...")
tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base")
@@ -71,7 +71,7 @@ def main():
if not args.interactive and not args.text:
infer = f"data/processed/{args.dataset}.csv"
infer_df = pd.read_csv(infer)
filename = f"outputs/inference/{args.model_path}_{args.task}_predictions_{args.dataset}.csv"
filename = f"outputs/inference/{args.model}_{args.task}_predictions_{args.dataset}.csv"
else:
infer_df = pd.DataFrame(columns=[args.text_column])
print("Entering interactive mode. Type 'exit()' to quit.")
@@ -80,7 +80,7 @@ def main():
if user_input.lower() == "exit()":
break
infer_df = pd.concat([infer_df, pd.DataFrame({args.text_column: [user_input]})], ignore_index=True)
filename = f"outputs/inference/{args.model_path}_{args.task}_predictions_interactive.csv"
filename = f"outputs/inference/{args.model}_{args.task}_predictions_interactive.csv"
infer_df.to_csv(filename, index=False)
infer = filename
@@ -93,8 +93,8 @@ def main():
if args.mode == "mtl":
model = Model().to(device)
print(f"Loading weights from {args.model_path}...")
model.load_state_dict(torch.load(f"outputs/{args.model_path}", map_location=device))
print(f"Loading weights from {args.model}...")
model.load_state_dict(torch.load(f"outputs/{args.model}", map_location=device))
model.eval()
active_tasks = ['bug_report', 'feature_request', 'aspect', 'aspect_sentiment']
else:
@@ -108,8 +108,8 @@ def main():
}
model = SingleTaskModel(args.task, task_classes[args.task]).to(device)
active_tasks = [args.task]
print(f"Loading weights from {args.model_path}...")
model.load_state_dict(torch.load(f"outputs/{args.model_path}", map_location=device))
print(f"Loading weights from {args.model}...")
model.load_state_dict(torch.load(f"outputs/{args.model}", map_location=device))
model.eval()
all_preds = {task: [] for task in active_tasks}

7
src/multitag.py
View File

@@ -6,6 +6,9 @@ from tkinter import ttk
import pandas as pd
import os
tagged_path = "data/uber_reviews_tagged.csv" # INPUT
sampled_path ="data/uber_reviews_sampled.csv" # OUTPUT
class MultiTag:
def __init__(self):
@@ -41,10 +44,6 @@ class MultiTag:
self.color_incomplete = "#003366"
self.color_complete = "#00AA00"
# Paths
tagged_path = "data/uber_reviews_tagged.csv"
sampled_path = "data/uber_reviews_sampled.csv"
if not os.path.exists(tagged_path):
print(f"Tagged file did not exist, making one at: {sampled_path}")
sampled_df = pd.read_csv(sampled_path, low_memory=False)

12
src/preprocess.py
View File

@@ -6,6 +6,11 @@
import pandas as pd
import re
# Should be arparse
input_file = "data/raw/uber_reviews.csv" # INPUT
output_file = "data/raw/uber_reviews_cleaned.csv" # OUTPUT
def clean_text(text) -> str:
"""Normalise review text by removing URLS, emails, excessive whitespace"""
if pd.isna(text):
@@ -109,7 +114,7 @@ def preprocess_uber_reviews(input_path, output_path):
rating_dist = df_clean['rating'].value_counts().sort_index()
for rating, count in rating_dist.items():
percentage = count / len(df_clean) * 100
print(f" {rating}{"✭"*rating}: {count:,} ({percentage:.1f}%)")
print(f" {rating}{'✭'*rating}: {count:,} ({percentage:.1f}%)")
print("\nWord count statistics:")
print(f" Mean: {df_clean['word_count'].mean():.1f} words")
@@ -129,16 +134,13 @@ def preprocess_uber_reviews(input_path, output_path):
for rating in [1,2,3,4,5]:
if len(df_clean[df_clean['rating'] == rating]) > 0:
sample = df_clean[df_clean['rating'] == rating].sample(min(2, len(df_clean[df_clean['rating'] == rating])))
print(f"\n{rating} {"✭" * rating} REVIEWS:")
print(f"\n{rating} {'✭' * rating} REVIEWS:")
for index, row in sample.iterrows():
print(f" • ({row['word_count']} words) {row['review'][:100]}")
return df_clean
if __name__ == "__main__":
input_file = "data/raw/uber_reviews.csv"
output_file = "data/raw/uber_reviews_cleaned.csv"
df_clean = preprocess_uber_reviews(input_file, output_file)
print("\nPreprocessing complete!")
print(f"Clean dataset: {len(df_clean):,} reviews ready for sampling")

24
src/sampler.py
View File

@@ -4,12 +4,16 @@ import numpy as np
print(pd.__version__)
print(np.__version__)
path = "data/raw/uber_reviews_cleaned.csv"
sampled_path = "data/raw/uber_reviews_sampled.csv"
original_path = "data/raw/uber_reviews.csv" ### only for distribution comparison
target_samples = 5000 # DEFAULT
cleaned_input = "data/raw/uber_reviews_cleaned.csv" # INPUT
sampled_path = "data/raw/uber_reviews_sampled.csv" # OUTPUT
### original path only for distribution comparison, comment out if using other sources ###
original_path = "data/raw/uber_reviews.csv"
class Sampler:
def __init__(self, data_path, target_samples):
self.target_samples = target_samples
self.data_path = data_path
# Default stratification method is based on original rating distribution
self.stratify_column = "rating"
@@ -168,8 +172,7 @@ class Sampler:
print(f" {rating}★: {count:,} ({pct:.1f}%)")
def main():
sampler = Sampler("data/raw/uber_reviews_cleaned.csv", target_samples=5000)
sampler = Sampler(cleaned_input, target_samples)
# Choose sampling strategy
print(f"\n{'='*50}")
@@ -178,24 +181,25 @@ def main():
print("1. get_stratified_sample() stratified by current distribution")
print("2. original_distribution_sample() stratified by the original data distribution")
print("3. get_keyword_boosted_sample() stratified using original distribution but also using a keyword dictionary")
print("4. sample_tiny_size() returns 200 samples to data/raw/tmp.csv")
choice = input("\nEnter choice (1-4): ").strip()
if choice == '1':
sample = sampler.get_stratified_sample()
sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv")
sampler.save_sample(sample, sampled_path)
elif choice == '2':
sample = sampler.original_distribution_sample()
sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv")
sampler.save_sample(sample, sampled_path)
elif choice == '3':
sample = sampler.sample_with_keywords()
sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv")
sampler.save_sample(sample, sampled_path)
elif choice == '4':
sample = sampler.sample_tiny_size()
sampler.save_sample(sample,"data/raw/uber_review_temp.csv")
sampler.save_sample(sample,"data/raw/tmp.csv")