mik/ReClass
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned notebooks, finished datalabelling

Browse Source
This commit is contained in:
charlie-rasberry
2026-02-16 12:36:29 +00:00
parent 8d3dee6d30
commit b88504725d
5 changed files with 199 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
README.md
View File

@@ -24,26 +24,28 @@ RECLASS is a multi-task learning system which uses a shared BERT encoder with ta
## Repository Structure ## Repository Structure
```
## Repository Structure
``` ```
6013/ 6013/
README.md README.md
requirements.txt .gitignore
multitag/ data/
data/ uber_reviews.csv # Raw dataset
uber_reviews.csv # Raw dataset uber_reviews_cleaned.csv # Preprocessed reviews
uber_reviews_cleaned.csv # Preprocessed reviews uber_reviews_sampled.csv # Stratified sample for annotation
uber_reviews_sampled.csv # Stratified sample for annotation uber_reviews_tagged.csv # Annotated reviews (in progress)
uber_reviews_tagged.csv # Annotated reviews (in progress) notebooks/
notebooks/ preprocessing_uber.ipynb # Preprocessing analysis
datasets_reviews.ipynb # Initial data exploration uber_cleaned.ipynb # Cleaned data verification
preprocessing_uber.ipynb # Preprocessing analysis src/
uber_cleaned.ipynb # Cleaned data verification preprocess.py # Text cleaning and filtering pipeline
src/ sampler.py # Stratified sampling strategies
preprocess.py # Text cleaning and filtering pipeline multitag.py # GUI annotation tool
sampler.py # Stratified sampling strategies train.py # Model training (in progress)
multitag.py # GUI annotation tool infer.py # Inference pipeline (in progress)
train.py # Model training (in progress) outputs/
infer.py # Inference pipeline (in progress) figures/
``` ```
## Current Progress ## Current Progress

98
notebooks/preprocessing_uber.ipynb
View File

@@ -1,8 +1,23 @@
{ {
"cells": [ "cells": [
{
"cell_type": "markdown",
"id": "f4474e0f",
"metadata": {},
"source": [
"# Preprocessing Requirements\n",
"## RECLASS\n",
"\n",
"**Purpose**: Ensure samples are consistent with the original dataset and find issues with current sampling/preprocessing methods.\n",
"\n",
"**Dataset**: Uber Customer Reviews from Google Play (Kaggle)\n",
"\n",
"---"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": 23,
"id": "470fe7c6-1614-4daf-879f-e6c399117c7b", "id": "470fe7c6-1614-4daf-879f-e6c399117c7b",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -13,7 +28,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 12, "execution_count": 24,
"id": "afe1168c", "id": "afe1168c",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -34,7 +49,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 13, "execution_count": 25,
"id": "b855045e-2dd1-4fa1-ab5a-8ce8b50b02ee", "id": "b855045e-2dd1-4fa1-ab5a-8ce8b50b02ee",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -45,7 +60,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 14, "execution_count": 26,
"id": "e7da1fb6-ede6-46c6-8fbd-fa491d3351c5", "id": "e7da1fb6-ede6-46c6-8fbd-fa491d3351c5",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -200,7 +215,7 @@
"4 4.486.10002 en in " "4 4.486.10002 en in "
] ]
}, },
"execution_count": 14, "execution_count": 26,
"metadata": {}, "metadata": {},
"output_type": "execute_result" "output_type": "execute_result"
} }
@@ -211,7 +226,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 15, "execution_count": 27,
"id": "5c02ec54-4583-4720-88c6-1110b52c3f88", "id": "5c02ec54-4583-4720-88c6-1110b52c3f88",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -235,7 +250,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 16, "execution_count": 28,
"id": "1da5d625-a4ba-49f8-8314-cc9e0f4ef96a", "id": "1da5d625-a4ba-49f8-8314-cc9e0f4ef96a",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -262,7 +277,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 17, "execution_count": 29,
"id": "1c97e396-8f05-4df7-bd0a-1bbecf6911b4", "id": "1c97e396-8f05-4df7-bd0a-1bbecf6911b4",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
@@ -272,7 +287,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 18, "execution_count": 30,
"id": "55324c94-4944-4844-b00e-dc08c8989f7b", "id": "55324c94-4944-4844-b00e-dc08c8989f7b",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -291,7 +306,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 19, "execution_count": 31,
"id": "c45959fe-3e23-4831-a41a-94c89892247f", "id": "c45959fe-3e23-4831-a41a-94c89892247f",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -326,7 +341,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 20, "execution_count": 32,
"id": "bf14e3db-a1b4-4fad-8102-b7ac25feeefa", "id": "bf14e3db-a1b4-4fad-8102-b7ac25feeefa",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -344,7 +359,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 21, "execution_count": 33,
"id": "8ccc07fa-9913-4047-ae17-35d2454eb059", "id": "8ccc07fa-9913-4047-ae17-35d2454eb059",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
@@ -357,66 +372,67 @@
"1 STAR REVIEWS:\n", "1 STAR REVIEWS:\n",
"==========================================\n", "==========================================\n",
"\n", "\n",
"Driver come very late than you can't do anthing pay and wait. Very bad service you can't cancil your when driver not come. Ubar charge money without service. This is very bad bad bad. So take local au...\n", "Many times driver cancelled the ride only because he didn't wanted to go to my destination and I was supposed to pay the cancellation fees without any reason.\n",
"(Length: 47.0 words)\n", "(Length: 28.0 words)\n",
"\n", "\n",
"I have uninstalled and reinstalled the app around 5 times over the course of the past 3 days. Every time I try to use the app, I get stuck in and endless reCaptcha loop. (I enter my phone number, solv...\n", "Their drivers are always finding new ways to outsmart customers.When Uber started initally, it was a pleasure to use their services.Now either when you book it says a few minutes n the moment you conf...\n",
"(Length: 66.0 words)\n", "(Length: 98.0 words)\n",
"\n", "\n",
"Thieves. Sent an Uber to my house in the middle of the night and wouldn't refund.\n", "terrible GPS system. takes you the long way everywhere. seriously, Waze, google maps and pretty much every other GPS shows faster routes. please fix this.\n",
"(Length: 16.0 words)\n", "(Length: 25.0 words)\n",
"\n", "\n",
"==========================================\n", "==========================================\n",
"2 STAR REVIEWS:\n", "2 STAR REVIEWS:\n",
"==========================================\n", "==========================================\n",
"\n", "\n",
"Your app is required to much space\n", "no helpline number, customer is unable to contect in case of emegrncy\n",
"(Length: 7.0 words)\n", "(Length: 12.0 words)\n",
"\n", "\n",
"I'm very disappointed. At first,I used Uber because it was far better than regular taxi. But I stopped using it because the application is very heavy and the drivers rarely reached my pinned locatio...\n", "Ghaantaa tum threk nhi kr skte u r just lying\n",
"(Length: 107.0 words)\n", "(Length: 10.0 words)\n",
"\n", "\n",
"nowhere to leave a tip!\n", "Nice application 😘😘\n",
"(Length: 5.0 words)\n", "(Length: 3.0 words)\n",
"\n", "\n",
"==========================================\n", "==========================================\n",
"3 STAR REVIEWS:\n", "3 STAR REVIEWS:\n",
"==========================================\n", "==========================================\n",
"\n", "\n",
"اوبر المدينة احيانا كويس .. بس لما يكون السائق باخر ملك ربي و تنتظر 14 دقيقه و بعدين يلغي و تصير دخلت بوقت الذروة المفروض يكون في تعويض .. زي لما تلغي انت .\n", "The app is good but I got charged for a cancelation because the driver was going to make me walk a block to go to him... what's the point in the app if I have to go to them\n",
"(Length: 34.0 words)\n", "(Length: 39.0 words)\n",
"\n", "\n",
"Good application\n", "Final amount to pay in cash doesn't always appear correct on app. You can't challenge the cost or question it. Example toll. They over charged by 60% of original cost and won't review it properly. Whe...\n",
"(Length: 2.0 words)\n", "(Length: 59.0 words)\n",
"\n", "\n",
"Toooslooow\n", "Location of the driver's car is not updated properly . I'm using android, and the location is keep being update all the time . Please fix this problem .\n",
"(Length: 1.0 words)\n", "(Length: 29.0 words)\n",
"\n", "\n",
"==========================================\n", "==========================================\n",
"4 STAR REVIEWS:\n", "4 STAR REVIEWS:\n",
"==========================================\n", "==========================================\n",
"\n", "\n",
"Help full\n", "Good\n",
"(Length: 2.0 words)\n", "(Length: 1.0 words)\n",
"\n", "\n",
"Won't allow me to change my payment details. Update: Problem solved.\n", "I like that app 😍🙃\n",
"(Length: 11.0 words)\n", "(Length: 5.0 words)\n",
"\n", "\n",
"Very good\n", "it is very difficult to contact the chief operator if there is any \n",
"(Length: 2.0 words)\n", "problem...we are not clear as to whom to contact if problem with uber driver\n",
"(Length: 27.0 words)\n",
"\n", "\n",
"==========================================\n", "==========================================\n",
"5 STAR REVIEWS:\n", "5 STAR REVIEWS:\n",
"==========================================\n", "==========================================\n",
"\n", "\n",
"Good driving skills\n", "I had a great uber experience at kolkata good experience.\n",
"(Length: 3.0 words)\n", "(Length: 10.0 words)\n",
"\n", "\n",
"Lovery\n", "Nice\n",
"(Length: 1.0 words)\n", "(Length: 1.0 words)\n",
"\n", "\n",
"Excellent experience\n", "It's an awesome aap\n",
"(Length: 2.0 words)\n" "(Length: 4.0 words)\n"
] ]
} }
], ],

96
notebooks/rating_distribution.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

19
notebooks/uber_cleaned.ipynb
View File

@@ -1,5 +1,24 @@
{ {
"cells": [ "cells": [
{
"cell_type": "markdown",
"id": "6318e936",
"metadata": {},
"source": [
"# Preprocessing Requirements\n",
"## RECLASS\n",
"\n",
"**Purpose**: Verify the output datasets.\n",
"\n",
"- `uber_reviews_cleaned.csv` - Full cleaned dataset\n",
"- `uber_reviews_sampled.csv` - tbc\n",
"- `uber_reviews_tagged.csv` - tbc ...Annotation progress\n",
"\n",
"**Dataset**: Uber Customer Reviews from Google Play (Kaggle)\n",
"\n",
"---"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 1, "execution_count": 1,

12
src/multitag.py
View File

@@ -2,6 +2,8 @@
# This app enables manual annotation of reviews in the Uber dataset, for training with # This app enables manual annotation of reviews in the Uber dataset, for training with
# to achieve review classifications with multi task deep learning # to achieve review classifications with multi task deep learning
# In another time I would have had much more tasks / classifications so mtl can perform better (that would mean better labelling),
#at least that is my prediction of why this may not be as good as I wanted
import tkinter as tk import tkinter as tk
from tkinter import ttk from tkinter import ttk
import pandas as pd import pandas as pd
@@ -47,8 +49,8 @@ class MultiTag:
self.color_complete = "#00AA00" self.color_complete = "#00AA00"
# Paths # Paths
tagged_path = "multitag/data/uber_reviews_tagged.csv" tagged_path = "data/uber_reviews_tagged.csv"
sampled_path = "multitag/data/uber_reviews_sampled.csv" sampled_path = "data/uber_reviews_sampled.csv"
# self.load_review_data("data/uber_reviews_sampled.csv") # self.load_review_data("data/uber_reviews_sampled.csv")
# self.load_review_data("data/uber_reviews_tagged.csv") # self.load_review_data("data/uber_reviews_tagged.csv")
if not os.path.exists(tagged_path): if not os.path.exists(tagged_path):
@@ -256,7 +258,7 @@ class MultiTag:
def submit_tag(self): def submit_tag(self):
self.review_data.at[self.current_review_index, "tagged"] = 1 self.review_data.at[self.current_review_index, "tagged"] = 1
self.save_tags("multitag/data/uber_reviews_tagged.csv") self.save_tags("data/uber_reviews_tagged.csv")
self.display_next_review() self.display_next_review()
def try_submit(self, event): def try_submit(self, event):
@@ -291,9 +293,9 @@ class MultiTag:
print(f"SESSION COMPLETE") print(f"SESSION COMPLETE")
print(f"{'='*50}") print(f"{'='*50}")
print(f"Total tagged: {tagged_count} / {len(self.review_data)}") print(f"Total tagged: {tagged_count} / {len(self.review_data)}")
print(f"Saved to: multitag/data/uber_reviews_tagged.csv") print(f"Saved to: data/uber_reviews_tagged.csv")
print(f"Bye (ʘ‿ʘ)╯") print(f"Bye (ʘ‿ʘ)╯")
self.save_tags("multitag/data/uber_reviews_tagged.csv") self.save_tags("data/uber_reviews_tagged.csv")
self.root.destroy() self.root.destroy()
def get_current_review_index(self): def get_current_review_index(self):