made some changes to the user embs, the model arch (classifier), added

some req to the text file, and wrote a comparison script

Author: danglingptr0x0

acaris_comparer.py

@@ -0,0 +1,62 @@
+from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, log_loss
+from sklearn.preprocessing import label_binarize
+from inferDistilBERT import InferACARISBERT
+from inferACARIS import InferACARIS
+import pandas as pd
+import numpy as np
+from tqdm import tqdm
+
+
+
+class ACARISComparer:
+	def __init__(self, baseline, acarisModel):
+		self.baseline = baseline
+		self.acarisModel = acarisModel
+
+	def compare(self, data):
+		uids = list(data["uid"])
+		contents = list(data["content"])
+		classLabels = ["neg", "neu", "pos"]
+		y_true = [classLabels.index(sentiment) for sentiment in data["sentiment"]]
+
+		baselineIN = [self.baseline.prepare_input(content) for content in contents]
+
+		baselinePreds = [self.baseline.predict(inputs=input_) for input_ in tqdm(baselineIN, desc="Predicting using baseline", ncols=100)]
+		acarisPreds = [self.acarisModel.predict(uids=[uid], contents=[content]) for uid, content in tqdm(zip(uids, contents), total=len(uids), desc="Predicting using ACARIS", ncols=100)]
+
+		probs1 = baselinePreds
+		probs2 = np.vstack([pred[1] for pred in acarisPreds])
+		y_pred1 = [np.argmax(prob) for prob in probs1]
+		y_pred2 = [np.argmax(prob) for prob in probs2]
+
+		print("Baseline metrics:")
+		y_true = np.array(y_true)
+		y_pred1 = np.array(y_pred1)
+		probs1 = np.concatenate(probs1)
+		self.compute_metrics(y_true, y_pred1, probs1, classLabels)
+
+		print("ACARIS metrics:")
+		self.compute_metrics(y_true, y_pred2, probs2, classLabels)
+
+	def compute_metrics(self, y_true, y_pred, y_probs, classLabels):
+		print(classification_report(y_true, y_pred, target_names=classLabels))
+
+		confMatrix = confusion_matrix(y_true, y_pred)
+		print(confMatrix)
+
+		y_true_bin = label_binarize(y_true, classes=range(len(classLabels)))
+		roc_auc = roc_auc_score(y_true_bin, y_probs, multi_class="ovr")
+		print(f"ROC AUC: {roc_auc}")
+
+		logloss = log_loss(y_true, y_probs)
+		print(f"Log loss: {logloss}")
+
+
+if __name__ == "__main__":
+	baseline = InferACARISBERT("ongknsro/ACARIS_BASELINE-DistilBERT-evalSpecimen1-batchSize32")
+	ACARISModel = InferACARIS("ongknsro/ACARIS-DistilBERT_MLPUserEmbs-iter1-batchSize32")
+
+	comparer = ACARISComparer(baseline, ACARISModel)
+
+	data = pd.read_csv("./datasets/test.csv", sep="|", encoding="utf-8")
+	comparer.compare(data)

dualTrainACARIS.py

@@ -0,0 +1,304 @@
+"""
+This mod fine-tunes a BERT model on the ACARIS dataset for comparison with ACARISMdl.
+"""
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+##################### !!! WORK IN PROGRESS !!! DO NOT USE !!! #####################
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+import torch
+from torch import nn
+from torch.utils.data import Dataset, DataLoader
+from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification, TrainingArguments, Trainer, AdamW, EarlyStoppingCallback, PreTrainedModel, DistilBertModel
+from transformers.modeling_outputs import SequenceClassifierOutput
+from datasets import load_dataset, Dataset
+import pandas as pd
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support, confusion_matrix, roc_auc_score
+import wandb
+import huggingface_hub
+import os
+import random
+import numpy as np
+
+config = {
+	"mdl": "distilbert-base-uncased",
+	"epochs": 5,
+	"batchSize": 14,
+	"maxLen": 512,
+	"warmupSteps": 0.1, # proportion of total steps, NOT absolute
+	"weightDecay": 0.02,
+	"outputDir": "./output",
+	"earlyStopping": True,
+	"earlyStoppingPatience": 2,
+	"dropout": 0.1,
+	"initlr": 5e-5,
+	"epsilon": 1e-8
+}
+
+wandb.init(project="MarkIII_ACARIS", entity="simtoonia", config=config)
+
+
+def lockSeed(seed):
+	random.seed(seed)
+	np.random.seed(seed)
+	torch.manual_seed(seed)
+	if torch.cuda.is_available():
+		torch.cuda.manual_seed_all(seed)
+		torch.backends.cudnn.deterministic = True
+
+#0 disabled, as determinism is not guaranteed and lowers performance
+#lockSeed(69) # setting a fixed seed for *some* reproducibility
+
+class DistilBertForMulticlassSequenceClassification(DistilBertForSequenceClassification):
+	def __init__(self, config):
+		super().__init__(config)
+
+	def forward(self, input_ids=None, attention_mask=None, head_mask=None, inputs_embeds=None, labels=None, output_attentions=None, output_hidden_states=None, return_dict=None):
+		return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+		outputs = self.distilbert(input_ids, attention_mask=attention_mask, head_mask=head_mask, inputs_embeds=inputs_embeds, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+
+		hidden_state = outputs[0]
+		pooled_output = hidden_state[:, 0]
+		pooled_output = self.pre_classifier(pooled_output)
+		pooled_output = nn.ReLU()(pooled_output)
+		pooled_output = self.dropout(pooled_output)
+		logits = self.classifier(pooled_output)
+
+		loss = None
+		if labels is not None:
+			lossFct = nn.CrossEntropyLoss()
+			loss = lossFct(logits.view(-1, self.num_labels), labels.view(-1))
+
+		if not return_dict:
+			output = (logits,) + outputs[2:]
+			return ((loss,) + output) if loss is not None else output
+
+		return SequenceClassifierOutput(loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions)
+
+
+
+class ACARISUserEmber:
+	def __init__(self, trainPath, valPath):
+		self.trainPath = trainPath
+		self.valPath = valPath
+		self.tokenizer = DistilBertTokenizerFast.from_pretrained(config["mdl"])
+		self.model = DistilBertForMulticlassSequenceClassification.from_pretrained(config["mdl"], num_labels=3, id2label={0: "neg", 1: "neu", 2: "pos"}, label2id={"neg": 0, "neu": 1, "pos": 2}, dropout=config["dropout"], attention_dropout=config["dropout"])
+
+	def read_data(self, path):
+		df = pd.read_csv(path, sep="|", usecols=["content", "sentiment"])
+		return Dataset.from_pandas(df)
+	
+	def tokenize_data(self, dataset):
+		sentMapping = {"pos": 2, "neg": 0, "neu": 1}
+		tokenized = dataset.map(
+			lambda x: {
+				**self.tokenizer(x["content"], truncation=True, padding="max_length", max_length=config["maxLen"]),
+				"labels": torch.tensor([sentMapping[sent] for sent in x["sentiment"]])
+			},
+			batched=True,
+			remove_columns=["content", "sentiment"]
+		)
+		tokenized.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+		return tokenized
+	
+	def get_data_loaders(self, trainDS, valDS):
+		trainLoader = DataLoader(trainDS, batch_size=config["batchSize"], shuffle=False)
+		valLoader = DataLoader(valDS, batch_size=config["batchSize"], shuffle=False)
+		return trainLoader, valLoader
+	
+	def compute_metrics(self, evalPred):
+		logits, labels = evalPred
+		preds = torch.argmax(torch.Tensor(logits), dim=1)
+		probs = torch.nn.functional.softmax(torch.Tensor(logits), dim=1)
+		precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=None)
+		accuracy = accuracy_score(labels, preds)
+		rocAUC = roc_auc_score(labels, probs, multi_class="ovr")
+		metrics = {
+			"accuracy": accuracy,
+			"roc_auc": rocAUC
+		}
+		metricNames = ["precision", "recall", "f1"]
+		labelNames = ["neg", "neu", "pos"]
+		for metricName, metricValue in zip(metricNames, [precision, recall, f1]):
+			for labelName, value in zip(labelNames, metricValue):
+				metrics[f"{metricName}_{labelName}"] = float(value)
+		return metrics
+	
+	def train(self):
+		trainDS = self.tokenize_data(self.read_data(self.trainPath))
+		valDS = self.tokenize_data(self.read_data(self.valPath))
+
+		totalSteps = len(trainDS) // config["batchSize"] * config["epochs"]
+		warmupSteps = int(totalSteps * config["warmupSteps"])
+		
+		trainingArgs = TrainingArguments(
+			output_dir=config["outputDir"],
+			num_train_epochs=config["epochs"],
+			per_device_train_batch_size=config["batchSize"],
+			per_device_eval_batch_size=config["batchSize"],
+			warmup_steps=warmupSteps,
+			weight_decay=config["weightDecay"],
+			logging_dir="./logs",
+			logging_steps=100,
+			learning_rate=config["initlr"],
+			evaluation_strategy="epoch",
+			save_strategy="epoch",
+			load_best_model_at_end=True,
+			metric_for_best_model="accuracy",
+			save_total_limit=5,
+			adam_epsilon=config["epsilon"],
+			report_to="wandb",
+			fp16=True
+		)
+		
+		trainer = Trainer(
+			model=self.model,
+			args=trainingArgs,
+			train_dataset=trainDS,
+			eval_dataset=valDS,
+			compute_metrics=self.compute_metrics,
+			callbacks=[EarlyStoppingCallback(early_stopping_patience=config["earlyStoppingPatience"])]
+		)
+		print(f"Number of parameters: {trainer.model.num_parameters()}")
+		print("Running eval ...")
+		trainer.evaluate()
+		print("Running training ...")
+		trainer.train()
+		print("Saving model ...")
+		trainer.save_model(config["outputDir"])
+
+
+
+class ACARISLabeler:
+	def __init__(self, trainPath, valPath):
+		self.trainPath = trainPath
+		self.valPath = valPath
+		self.tokenizer = DistilBertTokenizerFast.from_pretrained(config["mdl"])
+		self.model = DistilBertForMulticlassSequenceClassification.from_pretrained(config["mdl"], num_labels=3, id2label={0: "neg", 1: "neu", 2: "pos"}, label2id={"neg": 0, "neu": 1, "pos": 2}, dropout=config["dropout"], attention_dropout=config["dropout"])
+
+	def read_data(self, path):
+		df = pd.read_csv(path, sep="|", usecols=["content", "sentiment"])
+		return Dataset.from_pandas(df)
+	
+	def tokenize_data(self, dataset):
+		sentMapping = {"pos": 2, "neg": 0, "neu": 1}
+		tokenized = dataset.map(
+			lambda x: {
+				**self.tokenizer(x["content"], truncation=True, padding="max_length", max_length=config["maxLen"]),
+				"labels": torch.tensor([sentMapping[sent] for sent in x["sentiment"]])
+			},
+			batched=True,
+			remove_columns=["content", "sentiment"]
+		)
+		tokenized.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"])
+		return tokenized
+	
+	def get_data_loaders(self, trainDS, valDS):
+		trainLoader = DataLoader(trainDS, batch_size=config["batchSize"], shuffle=False)
+		valLoader = DataLoader(valDS, batch_size=config["batchSize"], shuffle=False)
+		return trainLoader, valLoader
+	
+	def compute_metrics(self, evalPred):
+		logits, labels = evalPred
+		preds = torch.argmax(torch.Tensor(logits), dim=1)
+		probs = torch.nn.functional.softmax(torch.Tensor(logits), dim=1)
+		precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average=None)
+		accuracy = accuracy_score(labels, preds)
+		rocAUC = roc_auc_score(labels, probs, multi_class="ovr")
+		metrics = {
+			"accuracy": accuracy,
+			"roc_auc": rocAUC
+		}
+		metricNames = ["precision", "recall", "f1"]
+		labelNames = ["neg", "neu", "pos"]
+		for metricName, metricValue in zip(metricNames, [precision, recall, f1]):
+			for labelName, value in zip(labelNames, metricValue):
+				metrics[f"{metricName}_{labelName}"] = float(value)
+		return metrics
+	
+	def train(self):
+		trainDS = self.tokenize_data(self.read_data(self.trainPath))
+		valDS = self.tokenize_data(self.read_data(self.valPath))
+
+		totalSteps = len(trainDS) // config["batchSize"] * config["epochs"]
+		warmupSteps = int(totalSteps * config["warmupSteps"])
+		
+		trainingArgs = TrainingArguments(
+			output_dir=config["outputDir"],
+			num_train_epochs=config["epochs"],
+			per_device_train_batch_size=config["batchSize"],
+			per_device_eval_batch_size=config["batchSize"],
+			warmup_steps=warmupSteps,
+			weight_decay=config["weightDecay"],
+			logging_dir="./logs",
+			logging_steps=100,
+			learning_rate=config["initlr"],
+			evaluation_strategy="epoch",
+			save_strategy="epoch",
+			load_best_model_at_end=True,
+			metric_for_best_model="accuracy",
+			save_total_limit=5,
+			adam_epsilon=config["epsilon"],
+			report_to="wandb",
+			fp16=True
+		)
+		
+		trainer = Trainer(
+			model=self.model,
+			args=trainingArgs,
+			train_dataset=trainDS,
+			eval_dataset=valDS,
+			compute_metrics=self.compute_metrics,
+			callbacks=[EarlyStoppingCallback(early_stopping_patience=config["earlyStoppingPatience"])]
+		)
+		print(f"Number of parameters: {trainer.model.num_parameters()}")
+		print("Running eval ...")
+		trainer.evaluate()
+		print("Running training ...")
+		trainer.train()
+		print("Saving model ...")
+		trainer.save_model(config["outputDir"])
+		
+		
+if __name__ == "__main__":
+	acaris_bert = ACARISBERT("./datasets/train.csv", "./datasets/val.csv")
+	acaris_bert.train()
+	wandb.finish()