From 7ba50ac23d10a57db0dfd0eb041031a03cd4a8eb Mon Sep 17 00:00:00 2001 From: charlie-rasberry Date: Fri, 17 Jul 2026 20:02:59 +0100 Subject: [PATCH] added: clearer input and output paths --- src/infer.py | 18 +++++++++--------- src/multitag.py | 7 +++---- src/preprocess.py | 14 ++++++++------ src/sampler.py | 26 +++++++++++++++----------- src/train.py | 2 +- 5 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/infer.py b/src/infer.py index 776bb33..6b5b4a0 100644 --- a/src/infer.py +++ b/src/infer.py @@ -35,7 +35,7 @@ np.random.seed(SEED) def parse_args(): parser = argparse.ArgumentParser(description="RECLASS, Multitask learning for review classification.") - parser.add_argument("--model_path", type=str, required=True, help=".pt file in outputs/") + parser.add_argument("--model", type=str, required=True, help=".pt file - just the filename not path") parser.add_argument("--task", type=str, default="all", choices=["all", "bug_report", "feature_request", "aspect", "aspect_sentiment"]) parser.add_argument("--interactive", action="store_true", help="Loops reading input until exit()") parser.add_argument("--text", action="store_true", help="Use command line text for input") @@ -63,7 +63,7 @@ def main(): else: print(f'{" "*15, "No GPUs available"}') print(f'{"="*50}\n') - print(f"Running inference on: outputs/{args.model_path} using data/processed/{args.dataset}.csv") + print(f"Running inference on: outputs/{args.model} using data/processed/{args.dataset}.csv") print("Loading model, tokenizer and datasets ...") tokenizer = AutoTokenizer.from_pretrained("FacebookAI/xlm-roberta-base") @@ -71,7 +71,7 @@ def main(): if not args.interactive and not args.text: infer = f"data/processed/{args.dataset}.csv" infer_df = pd.read_csv(infer) - filename = f"outputs/inference/{args.model_path}_{args.task}_predictions_{args.dataset}.csv" + filename = f"outputs/inference/{args.model}_{args.task}_predictions_{args.dataset}.csv" else: infer_df = pd.DataFrame(columns=[args.text_column]) print("Entering interactive mode. Type 'exit()' to quit.") @@ -80,7 +80,7 @@ def main(): if user_input.lower() == "exit()": break infer_df = pd.concat([infer_df, pd.DataFrame({args.text_column: [user_input]})], ignore_index=True) - filename = f"outputs/inference/{args.model_path}_{args.task}_predictions_interactive.csv" + filename = f"outputs/inference/{args.model}_{args.task}_predictions_interactive.csv" infer_df.to_csv(filename, index=False) infer = filename @@ -93,8 +93,8 @@ def main(): if args.mode == "mtl": model = Model().to(device) - print(f"Loading weights from {args.model_path}...") - model.load_state_dict(torch.load(f"outputs/{args.model_path}", map_location=device)) + print(f"Loading weights from {args.model}...") + model.load_state_dict(torch.load(f"outputs/{args.model}", map_location=device)) model.eval() active_tasks = ['bug_report', 'feature_request', 'aspect', 'aspect_sentiment'] else: @@ -108,8 +108,8 @@ def main(): } model = SingleTaskModel(args.task, task_classes[args.task]).to(device) active_tasks = [args.task] - print(f"Loading weights from {args.model_path}...") - model.load_state_dict(torch.load(f"outputs/{args.model_path}", map_location=device)) + print(f"Loading weights from {args.model}...") + model.load_state_dict(torch.load(f"outputs/{args.model}", map_location=device)) model.eval() all_preds = {task: [] for task in active_tasks} @@ -158,4 +158,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/multitag.py b/src/multitag.py index f9d91ce..b7fca38 100644 --- a/src/multitag.py +++ b/src/multitag.py @@ -6,6 +6,9 @@ from tkinter import ttk import pandas as pd import os +tagged_path = "data/uber_reviews_tagged.csv" # INPUT +sampled_path ="data/uber_reviews_sampled.csv" # OUTPUT + class MultiTag: def __init__(self): @@ -41,10 +44,6 @@ class MultiTag: self.color_incomplete = "#003366" self.color_complete = "#00AA00" - # Paths - tagged_path = "data/uber_reviews_tagged.csv" - sampled_path = "data/uber_reviews_sampled.csv" - if not os.path.exists(tagged_path): print(f"Tagged file did not exist, making one at: {sampled_path}") sampled_df = pd.read_csv(sampled_path, low_memory=False) diff --git a/src/preprocess.py b/src/preprocess.py index af95cab..43d75da 100644 --- a/src/preprocess.py +++ b/src/preprocess.py @@ -6,6 +6,11 @@ import pandas as pd import re +# Should be arparse +input_file = "data/raw/uber_reviews.csv" # INPUT +output_file = "data/raw/uber_reviews_cleaned.csv" # OUTPUT + + def clean_text(text) -> str: """Normalise review text by removing URLS, emails, excessive whitespace""" if pd.isna(text): @@ -109,7 +114,7 @@ def preprocess_uber_reviews(input_path, output_path): rating_dist = df_clean['rating'].value_counts().sort_index() for rating, count in rating_dist.items(): percentage = count / len(df_clean) * 100 - print(f" {rating}{"✭"*rating}: {count:,} ({percentage:.1f}%)") + print(f" {rating}{'✭'*rating}: {count:,} ({percentage:.1f}%)") print("\nWord count statistics:") print(f" Mean: {df_clean['word_count'].mean():.1f} words") @@ -129,16 +134,13 @@ def preprocess_uber_reviews(input_path, output_path): for rating in [1,2,3,4,5]: if len(df_clean[df_clean['rating'] == rating]) > 0: sample = df_clean[df_clean['rating'] == rating].sample(min(2, len(df_clean[df_clean['rating'] == rating]))) - print(f"\n{rating} {"✭" * rating} REVIEWS:") + print(f"\n{rating} {'✭' * rating} REVIEWS:") for index, row in sample.iterrows(): print(f" • ({row['word_count']} words) {row['review'][:100]}") return df_clean if __name__ == "__main__": - input_file = "data/raw/uber_reviews.csv" - output_file = "data/raw/uber_reviews_cleaned.csv" - df_clean = preprocess_uber_reviews(input_file, output_file) print("\nPreprocessing complete!") - print(f"Clean dataset: {len(df_clean):,} reviews ready for sampling") \ No newline at end of file + print(f"Clean dataset: {len(df_clean):,} reviews ready for sampling") diff --git a/src/sampler.py b/src/sampler.py index 152489d..7d261af 100644 --- a/src/sampler.py +++ b/src/sampler.py @@ -4,12 +4,16 @@ import numpy as np print(pd.__version__) print(np.__version__) -path = "data/raw/uber_reviews_cleaned.csv" -sampled_path = "data/raw/uber_reviews_sampled.csv" -original_path = "data/raw/uber_reviews.csv" ### only for distribution comparison +target_samples = 5000 # DEFAULT +cleaned_input = "data/raw/uber_reviews_cleaned.csv" # INPUT +sampled_path = "data/raw/uber_reviews_sampled.csv" # OUTPUT + +### original path only for distribution comparison, comment out if using other sources ### +original_path = "data/raw/uber_reviews.csv" + class Sampler: def __init__(self, data_path, target_samples): - + self.target_samples = target_samples self.data_path = data_path # Default stratification method is based on original rating distribution self.stratify_column = "rating" @@ -168,8 +172,7 @@ class Sampler: print(f" {rating}★: {count:,} ({pct:.1f}%)") def main(): - - sampler = Sampler("data/raw/uber_reviews_cleaned.csv", target_samples=5000) + sampler = Sampler(cleaned_input, target_samples) # Choose sampling strategy print(f"\n{'='*50}") @@ -178,26 +181,27 @@ def main(): print("1. get_stratified_sample() stratified by current distribution") print("2. original_distribution_sample() stratified by the original data distribution") print("3. get_keyword_boosted_sample() stratified using original distribution but also using a keyword dictionary") + print("4. sample_tiny_size() returns 200 samples to data/raw/tmp.csv") choice = input("\nEnter choice (1-4): ").strip() if choice == '1': sample = sampler.get_stratified_sample() - sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv") + sampler.save_sample(sample, sampled_path) elif choice == '2': sample = sampler.original_distribution_sample() - sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv") + sampler.save_sample(sample, sampled_path) elif choice == '3': sample = sampler.sample_with_keywords() - sampler.save_sample(sample, "data/raw/uber_reviews_sampled.csv") + sampler.save_sample(sample, sampled_path) elif choice == '4': sample = sampler.sample_tiny_size() - sampler.save_sample(sample,"data/raw/uber_review_temp.csv") + sampler.save_sample(sample,"data/raw/tmp.csv") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/train.py b/src/train.py index da5a0dd..bcc10f4 100644 --- a/src/train.py +++ b/src/train.py @@ -251,4 +251,4 @@ def main(): print(f"Peak GPU memory usage: {torch.cuda.max_memory_allocated(device) / (1024**3)} GB") if __name__ == "__main__": - main() \ No newline at end of file + main()