diff --git a/code/Functions.py b/code/Functions.py index 4f290d9..5ab3660 100644 --- a/code/Functions.py +++ b/code/Functions.py @@ -2,51 +2,62 @@ import math import re import json +import seaborn as sns import matplotlib.pyplot as plt from tqdm import tqdm +import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from sklearn.metrics import roc_curve, auc, precision_score, recall_score, f1_score +from sklearn.metrics import roc_curve, auc, f1_score, classification_report, precision_recall_curve from sentence_transformers import SentenceTransformer from StoreDataset import FileLoader, retrieve_to class LinearConfig: - pos_weight = torch.tensor(0.502773) + pos_weight = torch.tensor(0.65) criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - train_p = 0.8 - val_p = 0.1 + train_p = 0.75 + val_p = 0.15 - d_model = 128 + d_model_choices = [32, 64, 128, 256] num_epochs = 50 batch_num = 32 - learning_rate = 0.01 + learning_rate_choices = [0.1, 0.05, 0.01, 0.005, 0.001] stop_patience = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + learning_rate = 0.1 + d_model = 32 + class LogitConfig: - pos_weight = torch.tensor(0.5) + pos_weight = torch.tensor(0.7) criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) + add_conv_choices = [False, True] - train_p = 0.8 - val_p = 0.1 + train_p = 0.75 + val_p = 0.15 - d_model = 128 - conv_ch = 32 + d_model_choices = [32, 64, 128, 256] + conv_ch_choices = [32, 64] drop_out = 0.1 - num_epochs = 50 + num_epochs = 100 batch_num = 32 - learning_rate = 0.01 + learning_rate_choices = [0.1, 0.05, 0.01, 0.005, 0.001] patience = 10 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + add_conv = False + d_model = 64 + conv_ch = 64 + learning_rate = 0.05 + def count_files(dir): files = [f for f in os.listdir(dir)] return len(files) @@ -201,7 +212,12 @@ def run_batch(config, return total_loss -def plot_loss(history, is_linear): + +# ———————————————————————————————— +# plot graphs and caculate scores +# ———————————————————————————————— + +def plot_loss(history, is_linear, show, length=None): plt.figure(figsize=(8, 5)) plt.plot(history["running_loss"], label='running_loss', color='blue') plt.plot(history["val_loss"], label='val_loss', color='red', linestyle="dashed") @@ -211,18 +227,57 @@ def plot_loss(history, is_linear): plt.legend() if is_linear: - length = count_files(r"test_results\loss\linear") - plt.savefig(fr"test_results\loss\linear\linear_{length + 1}.pdf") + if not length: + length = count_files(r"test_results\loss\linear") + 1 + plt.savefig(fr"test_results\loss\linear\linear_{length}.pdf") else: - length = count_files(r"test_results\loss\logprob") - plt.savefig(fr"test_results\loss\logprob\logprob_{length + 1}.pdf") - plt.show() + if not length: + length = count_files(r"test_results\loss\logprob") + 1 + plt.savefig(fr"test_results\loss\logprob\logprob_{length}.pdf") + + if show: + plt.show() + +def calculate_threshold(model, config, val_loader): + + model.eval() + results = [] + targets = [] + + with torch.no_grad(): + for batch in val_loader: + batch = {k: v.to(config.device) for k, v in batch.items()} + + if model.is_linear: + y_pred = model(batch["cos_sim"], batch["entropy"]) + else: + y_pred = model( + batch["logprobs1"], + batch["logprobs2"], + batch["cos_sim"], + batch["entropy"] + ) + y_pred = torch.sigmoid(y_pred.squeeze(1)) + results.append(y_pred.tolist()) + targets.append(batch["labels"].tolist()) + + results = [item for res in results for item in res] + targets = [item for tar in targets for item in tar] + + precision, recall, thresholds = precision_recall_curve(targets, results) -def graph_roc_curve(config, test_loader, model, parameter_path: str): + f1_scores = 2 * (precision * recall) / (precision + recall + 1e-8) + f1_scores = f1_scores[:-1] + best_index = np.argmax(f1_scores) + best_threshold = thresholds[best_index] + + return best_threshold + +def test_model(config, test_loader, threshold, model, parameter_path, plot_distribution:bool = False): model.load_state_dict(torch.load(os.path.join("models", parameter_path))) - results, targets = run_batch( + y_pred, y_true = run_batch( config, test_loader, model, @@ -230,15 +285,38 @@ def graph_roc_curve(config, test_loader, model, parameter_path: str): "test", ) - fpr, tpr, threshold = roc_curve(targets, results) + if plot_distribution: + plt.close(1) + plt.figure(2) + sns.histplot(data=y_pred, bins=5, kde=True, color="teal", label="model output") + plt.xlim(0, 1) + + if input("Print 0.5 threshold? [y/n] ") == "y": + plt.axvline(0.5, color="b", linestyle="dashed", label="0.5 threshold") + if input("Print calculated threshold? [y/n] ") == "y": + plt.axvline(threshold, color="r", linestyle="dashed", label="calculated threshold") + plt.title("distribution of model output") + plt.legend() + + save = input("save? [y/n] ") == "y" + + if save: + name = input("input name: ") + plt.savefig(fr"test_results\{name}.png") + + plt.show() + + fpr, tpr, thre = roc_curve(y_true, y_pred) auroc = auc(fpr, tpr) - results = [0 if result < 0.5 else 1 for result in results] - f1 = f1_score(targets, results, average="binary") - prec = precision_score(targets, results) - recall = recall_score(targets, results) - print(f1) - print(prec) - print(recall) + y_pred = [0 if result < (threshold if threshold != -1 else 0.5) else 1 for result in y_pred] + f1 = f1_score(y_true, y_pred) + report = classification_report(y_true, y_pred, zero_division=0) + report_dict = classification_report(y_true, y_pred, output_dict=True, zero_division=0) + + return auroc, f1, fpr, tpr, report, report_dict + + +def graph_roc_curve(auroc, fpr, tpr, is_linear, length=None): plt.figure() plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % auroc) @@ -246,17 +324,17 @@ def graph_roc_curve(config, test_loader, model, parameter_path: str): plt.ylim([0.0, 1.05]) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') - plt.title(f'ROC Curve for {"linear model" if model.is_linear else "logprob model"}') + plt.title(f'ROC Curve for {"linear model" if is_linear else "logprob model"}') plt.legend() - if model.is_linear: - length = count_files(r"test_results\ROC\linear") - plt.savefig(fr"test_results\ROC\linear\{length + 1}.pdf") + if is_linear: + if not length: + length = count_files(r"test_results\ROC\linear") + 1 + plt.savefig(fr"test_results\ROC\linear\{length}.pdf") else: - length = count_files(r"test_results\ROC\logprob") - plt.savefig(fr"test_results\ROC\logprob\{length + 1}.pdf") - - plt.show() + if not length: + length = count_files(r"test_results\ROC\logprob") + 1 + plt.savefig(fr"test_results\ROC\logprob\{length}.pdf") if __name__ == "__main__": store_data(ans_dataset="TruthfulDataset.jsonl", cal_dataset="CalDataset.jsonl") \ No newline at end of file diff --git a/code/Layers.py b/code/Layers.py index 07bc891..f2b4b8a 100644 --- a/code/Layers.py +++ b/code/Layers.py @@ -21,7 +21,8 @@ def __init__(self): self.register_buffer("pe", pe) def forward(self, x): - return x + self.pe[:x.size(1), :].unsqueeze(0) + res = x + self.pe[:x.size(1), :].unsqueeze(0) + return res class Attention(nn.Module): def __init__(self, inner_feature): @@ -54,20 +55,26 @@ class LogitModel(nn.Module): def __init__(self, config): super().__init__() + self.con = config self.is_linear = False self.pe = PositionalEncoding() self.self_attention = Attention(inner_feature=config.d_model) - self.pooling_layer = nn.AdaptiveAvgPool1d(1) - self.dropout = nn.Dropout(p=config.drop_out) - self.fc_attention = nn.Linear(in_features=config.d_model, out_features=1) + if config.add_conv and config.conv_ch: + self.conv = nn.Conv2d(1, config.conv_ch, kernel_size=(3, 3)) + self.pooling_layer = nn.AdaptiveMaxPool2d((1, 1)) + self.dropout = nn.Dropout(p=config.drop_out) + self.fc_attention = nn.Linear(in_features=config.conv_ch, out_features=1) + elif not config.add_conv: + self.pooling_layer = nn.AdaptiveAvgPool1d(1) + self.dropout = nn.Dropout(p=config.drop_out) + self.fc_attention = nn.Linear(in_features=config.d_model, out_features=1) + else: + raise RuntimeError("add_conv not properly structured") + self.fc_final = nn.Linear(in_features=3, out_features=1) - # self.conv = nn.Conv2d(1, config.conv_ch, kernel_size=(3, 3)) - # self.pooling_layer = nn.AdaptiveMaxPool2d((1, 1)) - # self.dropout = nn.Dropout(p=config.drop_out) - # self.fc_attention = nn.Linear(in_features=config.conv_ch, out_features=1) def forward(self, resp1_logits, resp2_logits, cosine_sim, entropy): position1 = self.pe(resp1_logits) @@ -78,12 +85,15 @@ def forward(self, resp1_logits, resp2_logits, cosine_sim, entropy): position2, position2, ) - - pooled = self.pooling_layer(self_attention_values) - pooled = pooled.view(pooled.size(0), -1) - # conv_output = self.conv(self_attention_values.unsqueeze(1)) - # pooled = self.pooling_layer(conv_output).view(conv_output.size(0), conv_output.size(1)) + if self.con.add_conv: + conv_output = self.conv(self_attention_values.unsqueeze(1)) + pooled = self.pooling_layer(conv_output).view(conv_output.size(0), conv_output.size(1)) + elif not self.con.add_conv: + pooled = self.pooling_layer(self_attention_values) + pooled = pooled.view(pooled.size(0), -1) + else: + raise RuntimeError("add_conv not properly structured") dropped = self.dropout(pooled) attention_score = self.fc_attention(dropped) @@ -102,15 +112,26 @@ def __init__(self, config): self.is_linear = True self.fc1 = nn.Linear(in_features=2, out_features=config.d_model) - self.batch_norm = nn.BatchNorm1d(num_features=config.d_model) - self.fc2 = nn.Linear(in_features=config.d_model, out_features=1) + self.batch_norm1 = nn.BatchNorm1d(num_features=config.d_model) + self.activation1 = nn.ReLU() + + self.fc2 = nn.Linear(in_features=config.d_model, out_features=config.d_model) + self.batch_norm2 = nn.BatchNorm1d(num_features=config.d_model) + self.activation2 = nn.ReLU() + self.fc3 = nn.Linear(in_features=config.d_model, out_features=1) def forward(self, cosine_sim, entropy): combined = torch.cat([cosine_sim.unsqueeze(1), entropy.unsqueeze(1)], dim=1) - linear_output1 = self.fc1(combined) - normalized = self.batch_norm(linear_output1) - res = self.fc2(normalized) + x = self.fc1(combined) + x = self.batch_norm1(x) + x = self.activation1(x) + + x = self.fc2(x) + x = self.batch_norm2(x) + x = self.activation2(x) + res = self.fc3(x) + return res \ No newline at end of file diff --git a/code/LinearTrain.py b/code/LinearTrain.py index 2d5d52b..9ce13ac 100644 --- a/code/LinearTrain.py +++ b/code/LinearTrain.py @@ -1,49 +1,60 @@ import torch import torch.optim as optim from Layers import LinearModel -from DatasetLoader import linear_train_loader, linear_val_loader -from Functions import LinearConfig, plot_loss, run_batch, count_files - -config = LinearConfig() -device = config.device -length = count_files(r"models\linear") - -model = LinearModel(config) -optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) - -prev_loss = float('inf') -history = {'running_loss': [], 'val_loss': []} - -for epoch in range(config.num_epochs): - - running_loss = run_batch(config=config, - loader=linear_train_loader, - model=model, - optimizer=optimizer, - type="train", - epoch=epoch - ) - - val_loss = run_batch(config=config, - loader=linear_val_loader, - model=model, - optimizer=optimizer, - type="val", - ) - - running_loss /= len(linear_train_loader.dataset) - val_loss /= len(linear_val_loader.dataset) - history['running_loss'].append(running_loss) - history['val_loss'].append(val_loss) - - if val_loss < prev_loss: - prev_loss = val_loss - patience_count = 0 - torch.save(model.state_dict(), fr'models\linear\{length + 1}.pth') - else: - patience_count += 1 - if patience_count >= config.stop_patience: - print(f'Early stopping at epoch {epoch + 1}') - break - -plot_loss(history, is_linear=True) +from DatasetLoader import linear_train_loader, linear_val_loader, linear_test_loader +from Functions import LinearConfig, plot_loss, run_batch, test_model, graph_roc_curve + +def linear_train(config, model_pth): + + model = LinearModel(config) + optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + + prev_loss = float('inf') + history = {'running_loss': [], 'val_loss': []} + + for epoch in range(config.num_epochs): + + running_loss = run_batch(config=config, + loader=linear_train_loader, + model=model, + optimizer=optimizer, + type="train", + epoch=epoch + ) + + val_loss = run_batch(config=config, + loader=linear_val_loader, + model=model, + optimizer=optimizer, + type="val", + ) + + running_loss /= len(linear_train_loader.dataset) + val_loss /= len(linear_val_loader.dataset) + history['running_loss'].append(running_loss) + history['val_loss'].append(val_loss) + + if val_loss < prev_loss: + prev_loss = val_loss + patience_count = 0 + torch.save(model.state_dict(), fr'models\linear\{model_pth}') + else: + patience_count += 1 + if patience_count >= config.stop_patience: + print(f'Early stopping at epoch {epoch + 1}') + break + + return history + + +if __name__ == "__main__": + model_pth = input("model's parameter path: ") + + config = LinearConfig() + history = linear_train(config, model_pth) + model = LinearModel(config) + + plot_loss(history, is_linear=True, show=True) + + auroc, f1, fpr, tpr = test_model(config, linear_test_loader, model, fr"linear\{model_pth}") + graph_roc_curve(auroc, f1, fpr, tpr, is_linear=True) \ No newline at end of file diff --git a/code/LogprobTrain.py b/code/LogprobTrain.py index b4b6143..b519f95 100644 --- a/code/LogprobTrain.py +++ b/code/LogprobTrain.py @@ -1,52 +1,73 @@ import torch import torch.optim as optim +from torch.optim.lr_scheduler import ReduceLROnPlateau from Layers import LogitModel -from Functions import LogitConfig, plot_loss, run_batch, count_files -from DatasetLoader import logprob_train_loader, logprob_val_loader +from Functions import LogitConfig, plot_loss, run_batch, test_model, graph_roc_curve +from DatasetLoader import logprob_train_loader, logprob_val_loader, logprob_test_loader -config = LogitConfig() -device = config.device -length = count_files(r"models\logprob") +def logprob_train(config, model_pth): -model = LogitModel(config) -optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + model = LogitModel(config) + optimizer = optim.Adam(model.parameters(), lr=config.learning_rate) + scheduler = ReduceLROnPlateau( + optimizer, + mode='min', + patience=5, + factor=0.5 + ) -history = {"running_loss": [], "val_loss": []} -prev_loss = float("inf") + history = {"running_loss": [], "val_loss": []} + prev_loss = float("inf") -for epoch in range(config.num_epochs): - model.train() + for epoch in range(config.num_epochs): + model.train() - running_loss = run_batch( - config=config, - loader=logprob_train_loader, - model=model, - optimizer=optimizer, - type="train", - epoch=epoch - ) + running_loss = run_batch( + config=config, + loader=logprob_train_loader, + model=model, + optimizer=optimizer, + type="train", + epoch=epoch + ) - val_loss = run_batch( - config=config, - loader=logprob_val_loader, - model=model, - optimizer=optimizer, - type="val", - ) + val_loss = run_batch( + config=config, + loader=logprob_val_loader, + model=model, + optimizer=optimizer, + type="val", + ) + + running_loss /= len(logprob_train_loader.dataset) + val_loss /= len(logprob_val_loader.dataset) + history["running_loss"].append(running_loss) + history["val_loss"].append(val_loss) + + scheduler.step(val_loss) + + if val_loss < prev_loss: + prev_loss = val_loss + patience_count = 0 + torch.save(model.state_dict(), fr'models\logprob\{model_pth}') + else: + patience_count += 1 + if patience_count >= config.patience: + print(f"model stopped at {epoch} epochs.") + break + + return history + +if __name__ == "__main__": + model_pth = input("model's parameter path: ") + + config = LogitConfig() + history = logprob_train(config, model_pth) + model = LogitModel(config) + + plot_loss(history, is_linear=False, show=True) + + auroc, f1, fpr, tpr = test_model(config, logprob_test_loader, model, fr"logprob\{model_pth}") + graph_roc_curve(auroc, f1, fpr, tpr, is_linear=False) - running_loss /= len(logprob_train_loader.dataset) - val_loss /= len(logprob_val_loader.dataset) - history["running_loss"].append(running_loss) - history["val_loss"].append(val_loss) - - if val_loss < prev_loss: - prev_loss = val_loss - patience_count = 0 - torch.save(model.state_dict(), fr'models\logprob\{length + 1}.pth') - else: - patience_count += 1 - if patience_count >= config.patience: - print(f"model stopped at {epoch} epochs.") - break - -plot_loss(history, is_linear=False) \ No newline at end of file + \ No newline at end of file diff --git a/code/Main.py b/code/Main.py new file mode 100644 index 0000000..5b5e23f --- /dev/null +++ b/code/Main.py @@ -0,0 +1,118 @@ +import os +import json +from LinearTrain import linear_train +from LogprobTrain import logprob_train +from Functions import LogitConfig, LinearConfig, test_model, plot_loss, calculate_threshold +from DatasetLoader import logprob_test_loader, linear_test_loader, logprob_val_loader, linear_val_loader +from Layers import LogitModel, LinearModel + +def logprob_train_model(config, model, count, weighted): + history = logprob_train(config, f"{"weighted" if weighted else ""}{count}.pth") + threshold = calculate_threshold(model, config, logprob_val_loader) + auroc, f1, fpr, tpr, report, report_dict = test_model(config, logprob_test_loader, threshold, model, fr"logprob\{"weighted" if weighted else ""}{count}.pth") + + print(f"\n{report}\nauroc: {auroc}") + + with open(r"history\logprob_history.txt", "a", encoding="utf-8") as f: + f.write(f"\n{count}\n dimension: {config.d_model} lr: {config.learning_rate}{f" CNN channel: {config.conv_ch}" if config.add_conv else ""} auroc: {auroc} f1: {f1}") + + if auroc >= 0.60 and f1 >= 0.70: + plot_loss(history, is_linear=False, show=False, length=f"{"weighted" if weighted else ""}{count}") + + report_dict = report_dict["1.0"] + accuracy = report_dict["accuracy"] + del report_dict["support"] + report_dict["ID"] = f"{"weighted" if weighted else ""}{count}" + report_dict["accuracy"] = accuracy + report_dict["auroc"] = auroc + report_dict["d_model"] = config.d_model + report_dict["learning_rate"] = config.learning_rate + report_dict["conv_ch"] = config.conv_ch if config.add_conv else None + + with open(r"history\logprob_best.jsonl", "a", encoding="utf-8") as f: + json.dump(report_dict, f, ensure_ascii=False) + f.write("\n") + + else: + os.remove(fr"models\logprob\{"weighted" if weighted else ""}{count}.pth") + +def linear_train_model(config, model, count, weighted): + history = linear_train(config, f"{"weighted" if weighted else ""}{count}.pth") + threshold = calculate_threshold(model, config, linear_val_loader) + auroc, f1, fpr, tpr, report, report_dict = test_model(config, linear_test_loader, threshold, model, fr"linear\{"weighted" if weighted else ""}{count}.pth") + print(f"\n{report}\nauroc: {auroc}") + + with open(r"history\linear_history.txt", "a", encoding="utf-8") as f: + f.write(f"\n{count}\n dimension: {config.d_model} lr: {config.learning_rate} auroc: {auroc} f1: {f1}") + + if auroc >= 0.60 and f1 >= 0.70: + plot_loss(history, is_linear=True, show=False, length=f"{"weighted" if weighted else ""}{count}") + + report_dict = report_dict["1.0"] + del report_dict["support"] + report_dict["ID"] = f"{"weighted" if weighted else ""}{count}" + report_dict["auroc"] = auroc + report_dict["d_model"] = config.d_model + report_dict["learning_rate"] = config.learning_rate + + with open(r"history\linear_best.jsonl", "a", encoding="utf-8") as f: + json.dump(report_dict, f, ensure_ascii=False) + f.write("\n") + + else: + os.remove(fr"models\linear\{"weighted" if weighted else ""}{count}.pth") + +if __name__ == "__main__": + while True: + type = input("linear or logporb --> ") + weighted = input("weighted? [y/n] ") == "y" + + if type == "linear": + config = LinearConfig() + count = 1 + total = len(config.d_model_choices) * len (config.learning_rate_choices) + + for dimension in config.d_model_choices: + config.d_model = dimension + for lr in config.learning_rate_choices: + config.learning_rate = lr + print(f"{count} / {total}\n") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate}") + model = LinearModel(config) + linear_train_model(config, model, count, weighted) + count += 1 + break + + elif type == "logprob": + config = LogitConfig() + count = 1 + total = len(config.d_model_choices) * len(config.learning_rate_choices) * (1 + len(config.conv_ch_choices) if len(config.add_conv_choices) == 2 else 1) + + for is_conv in config.add_conv_choices: + print("----With CNN layer----\n" if config.add_conv else "----No CNN layer----\n") + config.add_conv = is_conv + for item in config.d_model_choices: + config.d_model = item + for lr in config.learning_rate_choices: + config.learning_rate = lr + if config.add_conv: + for i in config.conv_ch_choices: + config.conv_ch = i + + print(f"{count} / {total}\n") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate} CNN channel: {config.conv_ch}") + model = LogitModel(config) + logprob_train_model(config, model, count, weighted) + count += 1 + + else: + print(f"{count} / {total}") + print(f"model dimension: {config.d_model} learning rate: {config.learning_rate}") + model = LogitModel(config) + logprob_train_model(config, model, count, weighted) + count += 1 + break + + else: + print("\n---------input [linear] or [logprob]----------\n") + diff --git a/code/TestModel.py b/code/TestModel.py deleted file mode 100644 index 8284810..0000000 --- a/code/TestModel.py +++ /dev/null @@ -1,23 +0,0 @@ -from Functions import LogitConfig, LinearConfig, graph_roc_curve, count_files -from Layers import LogitModel, LinearModel -from DatasetLoader import logprob_test_loader, linear_test_loader - -config = LogitConfig() -model = LogitModel(config) -length = count_files(r"models\logprob") -graph_roc_curve( - config, - logprob_test_loader, - model, - fr"logprob\{length}.pth", -) - -config = LinearConfig() -model = LinearModel(config) -length = count_files(r"models\linear") -graph_roc_curve( - config, - linear_test_loader, - model, - fr"linear\{length}.pth", -) \ No newline at end of file diff --git a/code/__pycache__/Functions.cpython-314.pyc b/code/__pycache__/Functions.cpython-314.pyc index 547e8e6..bee1423 100644 Binary files a/code/__pycache__/Functions.cpython-314.pyc and b/code/__pycache__/Functions.cpython-314.pyc differ diff --git a/code/__pycache__/Layers.cpython-314.pyc b/code/__pycache__/Layers.cpython-314.pyc index 7dd9135..752dbf6 100644 Binary files a/code/__pycache__/Layers.cpython-314.pyc and b/code/__pycache__/Layers.cpython-314.pyc differ diff --git a/code/__pycache__/LinearTrain.cpython-314.pyc b/code/__pycache__/LinearTrain.cpython-314.pyc new file mode 100644 index 0000000..c9bfcbf Binary files /dev/null and b/code/__pycache__/LinearTrain.cpython-314.pyc differ diff --git a/code/__pycache__/LogprobTrain.cpython-314.pyc b/code/__pycache__/LogprobTrain.cpython-314.pyc new file mode 100644 index 0000000..367ed42 Binary files /dev/null and b/code/__pycache__/LogprobTrain.cpython-314.pyc differ diff --git a/code/test.py b/code/test.py new file mode 100644 index 0000000..2313959 --- /dev/null +++ b/code/test.py @@ -0,0 +1,269 @@ +import matplotlib.pyplot as plt +from Functions import test_model, calculate_threshold, LogitConfig, LinearConfig +from Layers import LogitModel, LinearModel +from DatasetLoader import logprob_test_loader, linear_test_loader, logprob_val_loader, linear_val_loader +from StoreDataset import FileLoader + +COLORS = ["b", "g", "r", "c", "m", "y", "k"] + +if __name__ == "__main__": + mode = input("logprob or linear or both --> ") + + if mode == "logprob": + + compare = False + color = 0 + + plt.figure(1) + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + + #initialize configuration + config = LogitConfig() + + #asks for file location + id = int(input("ID of model to test -->")) + weighted = input("weighted? [y/n] ") == "y" + location = f"{"weighted" if weighted else ""}{id}" + file = FileLoader(rf"..\history\logprob_best.jsonl") + + #checks for ID + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + # changes config for testing + if parameters["conv_ch"]: + config.add_conv = True + config.conv_ch = parameters["conv_ch"] + else: + config.add_conv = False + + config.d_model = parameters["d_model"] + + # Initialize model + model = LogitModel(config) + + plot_distribution = input("Plot model distribution: [y/n]") == "y" + cal_thresh = input("Calculate threshold? [y/n] ") == "y" + + # calculate f1 score and auroc score + auroc, f1, fpr, tpr, report, report_dict = test_model( + config, + logprob_test_loader, + calculate_threshold(model, config, logprob_val_loader) if cal_thresh else -1, + model, + parameter_path=fr"logprob\{location}.pth", + plot_distribution=plot_distribution + ) + + + print(report) + print(f"auroc score: {auroc}") + + if plot_distribution: + break + + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=f"{COLORS[color]}") + + # asks if user want to compare between logprob models + compare = input("Compare between models? [y/n] ") == "y" + color += 1 + if color >= 7: + color = 0 + + if not compare: + break + + if not plot_distribution: + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + save = input("save? [y/n] ") == "y" + + if save: + name = input("input name: ") + plt.savefig(fr"test_results\ROC\logprob\{name}.png") + + plt.show() + + + elif mode == "linear": + color = 0 + compare = False + + plt.figure(1) + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + config = LinearConfig() + id = int(input("ID of model to test -->")) + weighted = input("weighted? [y/n] ") == "y" + location = f"{"weighted" if weighted else ""}{id}" + + file = FileLoader(rf"..\history\linear_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + config.d_model = parameters["d_model"] + + model = LinearModel(config) + + + plot_distribution = input("Plot model distribution: [y/n]") == "y" + cal_thresh = input("Calculate threshold? [y/n] ") == "y" + + auroc, f1, fpr, tpr, report, report_dict = test_model( + config, + linear_test_loader, + calculate_threshold(model, config, logprob_val_loader) if cal_thresh else -1, + model, + parameter_path=fr"linear\{location}.pth", + plot_distribution=plot_distribution + ) + + print(report) + print(f"auroc score: {auroc}") + + if plot_distribution: + break + + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=COLORS[color]) + + compare = input("Compare between models? [y/n] ") == "y" + color += 1 + if color >= 7: + color = 0 + + if not compare: + break + if not plot_distribution: + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + + save = input("save? [y/n] ") == "y" + + if save: + name = input("input name: ") + plt.savefig(fr"test_results\ROC\linear\{name}.png") + + plt.show() + + + elif mode == "both": + + color = 0 + compare = False + plt.figure() + plt.plot([0, 1], color="grey", linestyle="dashed", label="random model") + + while True: + + config = LogitConfig() + + id = int(input("ID of logprob model to test -->")) + weighted = input("weighted? [y/n] ") == "y" + location = f"{"weighted" if weighted else ""}{id}" + file = FileLoader(rf"..\history\logprob_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + if parameters["conv_ch"]: + config.add_conv = True + config.conv_ch = parameters["conv_ch"] + else: + config.add_conv = False + + config.d_model = parameters["d_model"] + + model = LogitModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, logprob_test_loader, 0.5, model, parameter_path=fr"logprob\{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=f"{COLORS[color]}") + + print(report) + print(auroc) + + compare = input("Compare between logprob models? [y/n] ") == "y" + color += 1 + if color >= 7: + color = 0 + + if not compare: + break + + while True: + config = LinearConfig() + + id = int(input("ID of linear model to test -->")) + weighted = input("weighted? [y/n] ") == "y" + location = f"{"weighted" if weighted else ""}{id}" + file = FileLoader(rf"..\history\linear_best.jsonl") + + parameters = None + for item in file: + if item["ID"] == location: + parameters = item + break + if not parameters: + raise RuntimeError("ID not in file") + + config.d_model = parameters["d_model"] + + model = LinearModel(config) + + auroc, f1, fpr, tpr, report, report_dict = test_model(config, linear_test_loader, 0.5, model, parameter_path=fr"linear\{location}.pth") + label = input("model label: ") + plt.plot(fpr, tpr, label=f'{label} (AUC=%0.2f)' % auroc, color=f"{COLORS[color]}") + + print(report) + print(auroc) + + compare = input("Compare between models? [y/n] ") == "y" + + if not compare: + break + + plt.xlim([0.0, 1.0]) + plt.ylim([0.0, 1.05]) + plt.xlabel('False Positive Rate') + plt.ylabel('True Positive Rate') + plt.title(f'ROC Curves') + plt.legend() + + save = input("save? [y/n] ") == "y" + + if save: + name = input("input name: ") + plt.savefig(fr"test_results\ROC\{name}.png") + plt.show() + + else: + raise TypeError("Mode not logprob or linear or all") diff --git a/group_table.png b/group_table.png new file mode 100644 index 0000000..4b2ee40 Binary files /dev/null and b/group_table.png differ diff --git a/group_table_final.png b/group_table_final.png new file mode 100644 index 0000000..0fe6c8d Binary files /dev/null and b/group_table_final.png differ diff --git a/history.txt b/history.txt deleted file mode 100644 index 8c8be64..0000000 --- a/history.txt +++ /dev/null @@ -1,172 +0,0 @@ -hallucination count = 541 -non hallucination count = 275 -weight = 1: 1.96727 - -Trial 1: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 32 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - stop_patience = 10 - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 32 - drop_out = 0.2 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - - -trial 2: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - stop_patience = 10 - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - drop_out = 0.2 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -trial 3: - -class LinearConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - stop_patience = 10 - - -class LogitConfig: - - criterion = nn.BCELoss() - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - drop_out = 0.1 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - patience = 10 - -trial 4: - criterion = nn.BCELoss() - - train_p = 0.8 - val_p = 0.1 - - d_model = 64 - drop_out = 0.0 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -trial 5: - -class LinearConfig: - - pos_weight = torch.tensor(0.502773) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.75 - val_p = 0.125 - - d_model = 64 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.05 - stop_patience = 10 - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -class LogitConfig: - - pos_weight = torch.tensor(0.502773) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.8 - val_p = 0.1 - - d_model = 64 - drop_out = 0.0 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -trial 6: - -this time with convolutional network - -class LogitConfig: - - pos_weight = torch.tensor(0.5) - criterion = nn.BCEWithLogitsLoss(pos_weight=pos_weight) - - train_p = 0.8 - val_p = 0.1 - - d_model = 128 ---> conv_ch = 32 - drop_out = 0.1 - - num_epochs = 50 - batch_num = 32 - learning_rate = 0.01 - patience = 10 - -AUROC of 0.53 not good \ No newline at end of file diff --git a/history/linear_best.jsonl b/history/linear_best.jsonl new file mode 100644 index 0000000..d4649d3 --- /dev/null +++ b/history/linear_best.jsonl @@ -0,0 +1,36 @@ +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "1", "auroc": 0.6077681455039945, "d_model": 32, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "2", "auroc": 0.6274859765425803, "d_model": 32, "learning_rate": 0.05} +{"precision": 0.6708074534161491, "recall": 0.972972972972973, "f1-score": 0.7941176470588235, "ID": "3", "auroc": 0.625871154173041, "d_model": 32, "learning_rate": 0.01} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "4", "auroc": 0.6437191908890022, "d_model": 32, "learning_rate": 0.005} +{"precision": 0.6993006993006993, "recall": 0.9009009009009009, "f1-score": 0.7874015748031497, "ID": "5", "auroc": 0.6197518272989971, "d_model": 32, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "6", "auroc": 0.6507734149243584, "d_model": 64, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "7", "auroc": 0.6413394526602074, "d_model": 64, "learning_rate": 0.05} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "8", "auroc": 0.6277409484956655, "d_model": 64, "learning_rate": 0.01} +{"precision": 0.6792452830188679, "recall": 0.972972972972973, "f1-score": 0.8, "ID": "9", "auroc": 0.6291007989121197, "d_model": 64, "learning_rate": 0.005} +{"precision": 0.6728395061728395, "recall": 0.9819819819819819, "f1-score": 0.7985347985347986, "ID": "10", "auroc": 0.6163522012578616, "d_model": 64, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "11", "auroc": 0.6184769675335713, "d_model": 128, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "12", "auroc": 0.6510283868774435, "d_model": 128, "learning_rate": 0.05} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "14", "auroc": 0.6391296957334693, "d_model": 128, "learning_rate": 0.005} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "15", "auroc": 0.6328403875573687, "d_model": 128, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "16", "auroc": 0.6431242563318036, "d_model": 256, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "18", "auroc": 0.6119326874043856, "d_model": 256, "learning_rate": 0.01} +{"precision": 0.697986577181208, "recall": 0.9369369369369369, "f1-score": 0.8, "ID": "20", "auroc": 0.6073431922488526, "d_model": 256, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted1", "auroc": 0.6116777154513005, "d_model": 32, "learning_rate": 0.1} +{"precision": 0.7317073170731707, "recall": 0.8108108108108109, "f1-score": 0.7692307692307693, "ID": "weighted2", "auroc": 0.635985041645419, "d_model": 32, "learning_rate": 0.05} +{"precision": 0.7155172413793104, "recall": 0.7477477477477478, "f1-score": 0.7312775330396476, "ID": "weighted3", "auroc": 0.6325004249532551, "d_model": 32, "learning_rate": 0.01} +{"precision": 0.7107438016528925, "recall": 0.7747747747747747, "f1-score": 0.7413793103448276, "ID": "weighted4", "auroc": 0.6364099949005609, "d_model": 32, "learning_rate": 0.005} +{"precision": 0.711864406779661, "recall": 0.7567567567567568, "f1-score": 0.7336244541484717, "ID": "weighted5", "auroc": 0.6015638279789224, "d_model": 32, "learning_rate": 0.001} +{"precision": 0.7121212121212122, "recall": 0.8468468468468469, "f1-score": 0.7736625514403292, "ID": "weighted6", "auroc": 0.6010538840727521, "d_model": 64, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted7", "auroc": 0.6510283868774435, "d_model": 64, "learning_rate": 0.05} +{"precision": 0.6986301369863014, "recall": 0.918918918918919, "f1-score": 0.7937743190661478, "ID": "weighted8", "auroc": 0.6163522012578616, "d_model": 64, "learning_rate": 0.01} +{"precision": 0.6754966887417219, "recall": 0.918918918918919, "f1-score": 0.7786259541984732, "ID": "weighted9", "auroc": 0.6353901070882203, "d_model": 64, "learning_rate": 0.005} +{"precision": 0.7314814814814815, "recall": 0.7117117117117117, "f1-score": 0.7214611872146118, "ID": "weighted10", "auroc": 0.6569777324494306, "d_model": 64, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted11", "auroc": 0.6376848546659867, "d_model": 128, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted12", "auroc": 0.6653068162502125, "d_model": 128, "learning_rate": 0.05} +{"precision": 0.7378640776699029, "recall": 0.6846846846846847, "f1-score": 0.7102803738317757, "ID": "weighted13", "auroc": 0.6664966853646098, "d_model": 128, "learning_rate": 0.01} +{"precision": 0.6708074534161491, "recall": 0.972972972972973, "f1-score": 0.7941176470588235, "ID": "weighted14", "auroc": 0.6289308176100629, "d_model": 128, "learning_rate": 0.005} +{"precision": 0.7232142857142857, "recall": 0.7297297297297297, "f1-score": 0.726457399103139, "ID": "weighted15", "auroc": 0.667176610572837, "d_model": 128, "learning_rate": 0.001} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted16", "auroc": 0.6567227604963455, "d_model": 256, "learning_rate": 0.1} +{"precision": 0.676829268292683, "recall": 1.0, "f1-score": 0.8072727272727273, "ID": "weighted17", "auroc": 0.6299507054224036, "d_model": 256, "learning_rate": 0.05} +{"precision": 0.6923076923076923, "recall": 0.8918918918918919, "f1-score": 0.7795275590551181, "ID": "weighted19", "auroc": 0.610742818289988, "d_model": 256, "learning_rate": 0.005} +{"precision": 0.697841726618705, "recall": 0.8738738738738738, "f1-score": 0.776, "ID": "weighted20", "auroc": 0.6440591534931158, "d_model": 256, "learning_rate": 0.001} diff --git a/history/linear_history.txt b/history/linear_history.txt new file mode 100644 index 0000000..bdb1dee --- /dev/null +++ b/history/linear_history.txt @@ -0,0 +1,219 @@ +with 0.5 weight +1 + dimension: 32 lr: 0.1 auroc: 0.6195818459969403 f1: 0.6730769230769231 +2 + dimension: 32 lr: 0.05 auroc: 0.668706442291348 f1: 0.6926829268292682 +3 + dimension: 32 lr: 0.01 auroc: 0.6583375828658848 f1: 0.6382978723404256 +4 + dimension: 32 lr: 0.005 auroc: 0.6408295087540371 f1: 0.6321243523316062 +5 + dimension: 32 lr: 0.001 auroc: 0.6527281998980113 f1: 0.6382978723404256 +6 + dimension: 64 lr: 0.1 auroc: 0.5884752677205507 f1: 0.8072727272727273 +7 + dimension: 64 lr: 0.05 auroc: 0.6423593404725482 f1: 0.7368421052631579 +8 + dimension: 64 lr: 0.01 auroc: 0.6338602753697095 f1: 0.6421052631578947 +9 + dimension: 64 lr: 0.005 auroc: 0.6226415094339622 f1: 0.6428571428571429 +10 + dimension: 64 lr: 0.001 auroc: 0.6610572836987931 f1: 0.6829268292682927 +11 + dimension: 128 lr: 0.1 auroc: 0.6656467788543261 f1: 0.7239819004524887 +12 + dimension: 128 lr: 0.05 auroc: 0.6692163861975183 f1: 0.7207207207207207 +13 + dimension: 128 lr: 0.01 auroc: 0.6421893591704912 f1: 0.6321243523316062 +14 + dimension: 128 lr: 0.005 auroc: 0.6688764235934048 f1: 0.6926829268292682 +15 + dimension: 128 lr: 0.001 auroc: 0.6447390787013428 f1: 0.6288659793814433 +16 + dimension: 256 lr: 0.1 auroc: 0.6600373958864526 f1: 0.7985347985347986 +17 + dimension: 256 lr: 0.05 auroc: 0.6357300696923339 f1: 0.7985347985347986 +18 + dimension: 256 lr: 0.01 auroc: 0.6450790413054565 f1: 0.6338797814207651 +19 + dimension: 256 lr: 0.005 auroc: 0.6693863674995751 f1: 0.6111111111111112 +20 + dimension: 256 lr: 0.001 auroc: 0.6342002379738229 f1: 0.6288659793814433 + +without weights: +1 + dimension: 32 lr: 0.1 auroc: 0.6343702192758797 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.6583375828658847 f1: 0.8072727272727273 +3 + dimension: 32 lr: 0.01 auroc: 0.6591874893761687 f1: 0.775 +4 + dimension: 32 lr: 0.005 auroc: 0.6554479007309196 f1: 0.7935222672064778 +5 + dimension: 32 lr: 0.001 auroc: 0.6568077511473739 f1: 0.75 +6 + dimension: 64 lr: 0.1 auroc: 0.65068842427333 f1: 0.808 + +with weight +1 + dimension: 32 lr: 0.1 auroc: 0.6503484616692164 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.6539180690124087 f1: 0.7466666666666667 +3 + dimension: 32 lr: 0.01 auroc: 0.6673465918748939 f1: 0.8072727272727273 +4 + dimension: 32 lr: 0.005 auroc: 0.6659867414584396 f1: 0.7214611872146118 +5 + dimension: 32 lr: 0.001 auroc: 0.6598674145843958 f1: 0.8072727272727273 +6 + dimension: 64 lr: 0.1 auroc: 0.663437021927588 f1: 0.8072727272727273 +7 + dimension: 64 lr: 0.05 auroc: 0.6646268910419854 f1: 0.8 +8 + dimension: 64 lr: 0.01 auroc: 0.6678565357810641 f1: 0.8072727272727273 +9 + dimension: 64 lr: 0.005 auroc: 0.6236613972463029 f1: 0.8072727272727273 +10 + dimension: 64 lr: 0.001 auroc: 0.6539180690124087 f1: 0.8 +11 + dimension: 128 lr: 0.1 auroc: 0.6693863674995751 f1: 0.8072727272727273 +12 + dimension: 128 lr: 0.05 auroc: 0.6664966853646099 f1: 0.8072727272727273 +13 + dimension: 128 lr: 0.01 auroc: 0.6673465918748939 f1: 0.7985074626865671 +14 + dimension: 128 lr: 0.005 auroc: 0.6707462179160293 f1: 0.591715976331361 +15 + dimension: 128 lr: 0.001 auroc: 0.6583375828658848 f1: 0.8029739776951673 +16 + dimension: 256 lr: 0.1 auroc: 0.6564677885432603 f1: 0.7970479704797048 +17 + dimension: 256 lr: 0.05 auroc: 0.6656467788543261 f1: 0.8072727272727273 +18 + dimension: 256 lr: 0.01 auroc: 0.6697263301036886 f1: 0.8072727272727273 +19 + dimension: 256 lr: 0.005 auroc: 0.6413394526602074 f1: 0.7819548872180451 +20 + dimension: 256 lr: 0.001 auroc: 0.669556348801632 f1: 0.6310160427807486 +1 + dimension: 32 lr: 0.1 auroc: 0.6646268910419854 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.6707462179160293 f1: 0.8072727272727273 +3 + dimension: 32 lr: 0.01 auroc: 0.6522182559918408 f1: 0.8072727272727273 +4 + dimension: 32 lr: 0.005 auroc: 0.6503484616692163 f1: 0.8072727272727273 +5 + dimension: 32 lr: 0.001 auroc: 0.6600373958864525 f1: 0.8072727272727273 +6 + dimension: 64 lr: 0.1 auroc: 0.658167601563828 f1: 0.8045112781954887 +7 + dimension: 64 lr: 0.05 auroc: 0.6692163861975183 f1: 0.8072727272727273 +8 + dimension: 64 lr: 0.01 auroc: 0.6668366479687236 f1: 0.7951807228915663 +9 + dimension: 64 lr: 0.005 auroc: 0.6695563488016318 f1: 0.7969924812030075 +10 + dimension: 64 lr: 0.001 auroc: 0.6573176950535442 f1: 0.8072727272727273 +11 + dimension: 128 lr: 0.1 auroc: 0.6554479007309196 f1: 0.8072727272727273 +12 + dimension: 128 lr: 0.05 auroc: 0.6607173210946795 f1: 0.8072727272727273 +13 + dimension: 128 lr: 0.01 auroc: 0.669556348801632 f1: 0.8072727272727273 +14 + dimension: 128 lr: 0.005 auroc: 0.664966853646099 f1: 0.8072727272727273 +15 + dimension: 128 lr: 0.001 auroc: 0.6437191908890023 f1: 0.8072727272727273 +16 + dimension: 256 lr: 0.1 auroc: 0.6630970593234744 f1: 0.8072727272727273 +17 + dimension: 256 lr: 0.05 auroc: 0.6585075641679417 f1: 0.8072727272727273 +18 + dimension: 256 lr: 0.01 auroc: 0.669386367499575 f1: 0.8072727272727273 +19 + dimension: 256 lr: 0.005 auroc: 0.6651368349481558 f1: 0.8029739776951673 +20 + dimension: 256 lr: 0.001 auroc: 0.6644569097399287 f1: 0.8072727272727273 + +with weights 3 MLP: +1 + dimension: 32 lr: 0.1 auroc: 0.6077681455039945 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.6274859765425803 f1: 0.8072727272727273 +3 + dimension: 32 lr: 0.01 auroc: 0.625871154173041 f1: 0.7941176470588235 +4 + dimension: 32 lr: 0.005 auroc: 0.6437191908890022 f1: 0.8072727272727273 +5 + dimension: 32 lr: 0.001 auroc: 0.6197518272989971 f1: 0.7874015748031497 +6 + dimension: 64 lr: 0.1 auroc: 0.6507734149243584 f1: 0.8072727272727273 +7 + dimension: 64 lr: 0.05 auroc: 0.6413394526602074 f1: 0.8072727272727273 +8 + dimension: 64 lr: 0.01 auroc: 0.6277409484956655 f1: 0.8072727272727273 +9 + dimension: 64 lr: 0.005 auroc: 0.6291007989121197 f1: 0.8 +10 + dimension: 64 lr: 0.001 auroc: 0.6163522012578616 f1: 0.7985347985347986 +11 + dimension: 128 lr: 0.1 auroc: 0.6184769675335713 f1: 0.8072727272727273 +12 + dimension: 128 lr: 0.05 auroc: 0.6510283868774435 f1: 0.8072727272727273 +13 + dimension: 128 lr: 0.01 auroc: 0.5510793812680604 f1: 0.8072727272727273 +14 + dimension: 128 lr: 0.005 auroc: 0.6391296957334693 f1: 0.8072727272727273 +15 + dimension: 128 lr: 0.001 auroc: 0.6328403875573687 f1: 0.8072727272727273 +16 + dimension: 256 lr: 0.1 auroc: 0.6431242563318036 f1: 0.8072727272727273 +17 + dimension: 256 lr: 0.05 auroc: 0.5905150433452321 f1: 0.8072727272727273 +18 + dimension: 256 lr: 0.01 auroc: 0.6119326874043856 f1: 0.8072727272727273 +19 + dimension: 256 lr: 0.005 auroc: 0.584565697773245 f1: 0.8072727272727273 +20 + dimension: 256 lr: 0.001 auroc: 0.6073431922488526 f1: 0.8 +1 + dimension: 32 lr: 0.1 auroc: 0.6116777154513005 f1: 0.8072727272727273 +2 + dimension: 32 lr: 0.05 auroc: 0.635985041645419 f1: 0.7692307692307693 +3 + dimension: 32 lr: 0.01 auroc: 0.6325004249532551 f1: 0.7312775330396476 +4 + dimension: 32 lr: 0.005 auroc: 0.6364099949005609 f1: 0.7413793103448276 +5 + dimension: 32 lr: 0.001 auroc: 0.6015638279789224 f1: 0.7336244541484717 +6 + dimension: 64 lr: 0.1 auroc: 0.6010538840727521 f1: 0.7736625514403292 +7 + dimension: 64 lr: 0.05 auroc: 0.6510283868774435 f1: 0.8072727272727273 +8 + dimension: 64 lr: 0.01 auroc: 0.6163522012578616 f1: 0.7937743190661478 +9 + dimension: 64 lr: 0.005 auroc: 0.6353901070882203 f1: 0.7786259541984732 +10 + dimension: 64 lr: 0.001 auroc: 0.6569777324494306 f1: 0.7214611872146118 +11 + dimension: 128 lr: 0.1 auroc: 0.6376848546659867 f1: 0.8072727272727273 +12 + dimension: 128 lr: 0.05 auroc: 0.6653068162502125 f1: 0.8072727272727273 +13 + dimension: 128 lr: 0.01 auroc: 0.6664966853646098 f1: 0.7102803738317757 +14 + dimension: 128 lr: 0.005 auroc: 0.6289308176100629 f1: 0.7941176470588235 +15 + dimension: 128 lr: 0.001 auroc: 0.667176610572837 f1: 0.726457399103139 +16 + dimension: 256 lr: 0.1 auroc: 0.6567227604963455 f1: 0.8072727272727273 +17 + dimension: 256 lr: 0.05 auroc: 0.6299507054224036 f1: 0.8072727272727273 +18 + dimension: 256 lr: 0.01 auroc: 0.5986741458439571 f1: 0.7871485943775101 +19 + dimension: 256 lr: 0.005 auroc: 0.610742818289988 f1: 0.7795275590551181 +20 + dimension: 256 lr: 0.001 auroc: 0.6440591534931158 f1: 0.776 \ No newline at end of file diff --git a/history/logprob_best.jsonl b/history/logprob_best.jsonl new file mode 100644 index 0000000..3478d58 --- /dev/null +++ b/history/logprob_best.jsonl @@ -0,0 +1,9 @@ +{"precision": 0.7014925373134329, "recall": 0.8545454545454545, "f1-score": 0.7704918032786885, "ID": "1", "auroc": 0.6409090909090909, "d_model": 32, "learning_rate": 0.1, "conv_ch": null} +{"precision": 0.6626506024096386, "recall": 1.0, "f1-score": 0.7971014492753623, "ID": "14", "auroc": 0.605844155844156, "d_model": 128, "learning_rate": 0.005, "conv_ch": null} +{"precision": 0.6666666666666666, "recall": 0.9090909090909091, "f1-score": 0.7692307692307693, "ID": "39", "auroc": 0.6233766233766234, "d_model": 64, "learning_rate": 0.001, "conv_ch": 32} +{"precision": 0.696969696969697, "recall": 0.8518518518518519, "f1-score": 0.7666666666666667, "ID": "weighted10", "auroc": 0.6071428571428571, "d_model": 64, "learning_rate": 0.001, "conv_ch": null} +{"precision": 0.7014925373134329, "recall": 0.8703703703703703, "f1-score": 0.7768595041322314, "ID": "weighted30", "auroc": 0.6342592592592593, "d_model": 32, "learning_rate": 0.001, "conv_ch": 64} +{"precision": 0.6714285714285714, "recall": 0.8703703703703703, "f1-score": 0.7580645161290323, "ID": "weighted38", "auroc": 0.6018518518518519, "d_model": 64, "learning_rate": 0.005, "conv_ch": 64} +{"precision": 0.696969696969697, "recall": 0.8518518518518519, "f1-score": 0.7666666666666667, "ID": "weighted45", "auroc": 0.6078042328042328, "d_model": 128, "learning_rate": 0.01, "conv_ch": 32} +{"precision": 0.6956521739130435, "recall": 0.8888888888888888, "f1-score": 0.7804878048780488, "ID": "weighted49", "auroc": 0.6507936507936508, "d_model": 128, "learning_rate": 0.001, "conv_ch": 32} +{"precision": 0.6585365853658537, "recall": 1.0, "f1-score": 0.7941176470588235, "ID": "weighted58", "auroc": 0.6011904761904762, "d_model": 256, "learning_rate": 0.005, "conv_ch": 64} diff --git a/history/logprob_history.txt b/history/logprob_history.txt new file mode 100644 index 0000000..b5068e7 --- /dev/null +++ b/history/logprob_history.txt @@ -0,0 +1,326 @@ + +logprob without weight + +1 + dimension: 32 lr: 0.1 auroc: 0.6058441558441559 f1: 0.7971014492753623 +1 + dimension: 32 lr: 0.1 auroc: 0.6409090909090909 f1: 0.7704918032786885 +2 + dimension: 32 lr: 0.05 auroc: 0.5285714285714286 f1: 0.7971014492753623 +3 + dimension: 32 lr: 0.01 auroc: 0.5 f1: 0.7971014492753623 +4 + dimension: 32 lr: 0.005 auroc: 0.525974025974026 f1: 0.7611940298507462 +5 + dimension: 32 lr: 0.001 auroc: 0.5753246753246752 f1: 0.8029197080291971 +6 + dimension: 64 lr: 0.1 auroc: 0.5376623376623377 f1: 0.7619047619047619 +7 + dimension: 64 lr: 0.05 auroc: 0.5467532467532467 f1: 0.7941176470588235 +8 + dimension: 64 lr: 0.01 auroc: 0.4792207792207792 f1: 0.7971014492753623 +9 + dimension: 64 lr: 0.005 auroc: 0.5064935064935064 f1: 0.8120300751879699 +10 + dimension: 64 lr: 0.001 auroc: 0.5993506493506493 f1: 0.7971014492753623 +11 + dimension: 128 lr: 0.1 auroc: 0.5922077922077922 f1: 0.7559055118110236 +12 + dimension: 128 lr: 0.05 auroc: 0.5623376623376624 f1: 0.7846153846153846 +13 + dimension: 128 lr: 0.01 auroc: 0.5149350649350649 f1: 0.7751937984496124 +14 + dimension: 128 lr: 0.005 auroc: 0.605844155844156 f1: 0.7971014492753623 +15 + dimension: 128 lr: 0.001 auroc: 0.4474025974025974 f1: 0.7971014492753623 +16 + dimension: 256 lr: 0.1 auroc: 0.5272727272727272 f1: 0.7971014492753623 +17 + dimension: 256 lr: 0.05 auroc: 0.5850649350649351 f1: 0.7424242424242424 +18 + dimension: 256 lr: 0.01 auroc: 0.5240259740259741 f1: 0.7794117647058824 +19 + dimension: 256 lr: 0.005 auroc: 0.5915584415584416 f1: 0.7971014492753623 +20 + dimension: 256 lr: 0.001 auroc: 0.5220779220779221 f1: 0.7971014492753623 +21 + dimension: 32 lr: 0.1 CNN channel: 32 auroc: 0.5363636363636364 f1: 0.8 +22 + dimension: 32 lr: 0.1 CNN channel: 64 auroc: 0.5461038961038961 f1: 0.7751937984496124 +23 + dimension: 32 lr: 0.05 CNN channel: 32 auroc: 0.4980519480519481 f1: 0.7703703703703704 +24 + dimension: 32 lr: 0.05 CNN channel: 64 auroc: 0.5201298701298701 f1: 0.7794117647058824 +25 + dimension: 32 lr: 0.01 CNN channel: 32 auroc: 0.5116883116883116 f1: 0.7883211678832117 +26 + dimension: 32 lr: 0.01 CNN channel: 64 auroc: 0.4831168831168831 f1: 0.7971014492753623 +27 + dimension: 32 lr: 0.005 CNN channel: 32 auroc: 0.5285714285714286 f1: 0.7971014492753623 +28 + dimension: 32 lr: 0.005 CNN channel: 64 auroc: 0.44999999999999996 f1: 0.7971014492753623 +29 + dimension: 32 lr: 0.001 CNN channel: 32 auroc: 0.5740259740259741 f1: 0.7883211678832117 +30 + dimension: 32 lr: 0.001 CNN channel: 64 auroc: 0.4850649350649351 f1: 0.7971014492753623 +31 + dimension: 64 lr: 0.1 CNN channel: 32 auroc: 0.5285714285714285 f1: 0.7971014492753623 +32 + dimension: 64 lr: 0.1 CNN channel: 64 auroc: 0.5487012987012987 f1: 0.7424242424242424 +33 + dimension: 64 lr: 0.05 CNN channel: 32 auroc: 0.5272727272727273 f1: 0.7971014492753623 +34 + dimension: 64 lr: 0.05 CNN channel: 64 auroc: 0.5292207792207793 f1: 0.7971014492753623 +35 + dimension: 64 lr: 0.01 CNN channel: 32 auroc: 0.5525974025974025 f1: 0.7751937984496124 +36 + dimension: 64 lr: 0.01 CNN channel: 64 auroc: 0.5233766233766234 f1: 0.7971014492753623 +37 + dimension: 64 lr: 0.005 CNN channel: 32 auroc: 0.5428571428571429 f1: 0.7971014492753623 +38 + dimension: 64 lr: 0.005 CNN channel: 64 auroc: 0.5142857142857142 f1: 0.8059701492537313 +39 + dimension: 64 lr: 0.001 CNN channel: 32 auroc: 0.6233766233766234 f1: 0.7692307692307693 +40 + dimension: 64 lr: 0.001 CNN channel: 64 auroc: 0.5889610389610389 f1: 0.7971014492753623 +41 + dimension: 128 lr: 0.1 CNN channel: 32 auroc: 0.5285714285714285 f1: 0.7611940298507462 +42 + dimension: 128 lr: 0.1 CNN channel: 64 auroc: 0.4967532467532467 f1: 0.7971014492753623 +43 + dimension: 128 lr: 0.05 CNN channel: 32 auroc: 0.5454545454545454 f1: 0.7971014492753623 +44 + dimension: 128 lr: 0.05 CNN channel: 64 auroc: 0.5175324675324675 f1: 0.7971014492753623 +45 + dimension: 128 lr: 0.01 CNN channel: 32 auroc: 0.5493506493506494 f1: 0.7703703703703704 +46 + dimension: 128 lr: 0.01 CNN channel: 64 auroc: 0.5642857142857143 f1: 0.7971014492753623 +47 + dimension: 128 lr: 0.005 CNN channel: 32 auroc: 0.49155844155844164 f1: 0.7971014492753623 +48 + dimension: 128 lr: 0.005 CNN channel: 64 auroc: 0.5474025974025973 f1: 0.7971014492753623 +49 + dimension: 128 lr: 0.001 CNN channel: 32 auroc: 0.5279220779220779 f1: 0.7971014492753623 +50 + dimension: 128 lr: 0.001 CNN channel: 64 auroc: 0.49870129870129865 f1: 0.7883211678832117 +51 + dimension: 256 lr: 0.1 CNN channel: 32 auroc: 0.5889610389610389 f1: 0.752 +52 + dimension: 256 lr: 0.1 CNN channel: 64 auroc: 0.5344155844155845 f1: 0.765625 +53 + dimension: 256 lr: 0.05 CNN channel: 32 auroc: 0.557142857142857 f1: 0.7971014492753623 +54 + dimension: 256 lr: 0.05 CNN channel: 64 auroc: 0.5480519480519481 f1: 0.765625 +55 + dimension: 256 lr: 0.01 CNN channel: 32 auroc: 0.5506493506493506 f1: 0.7971014492753623 +56 + dimension: 256 lr: 0.01 CNN channel: 64 auroc: 0.5318181818181819 f1: 0.7971014492753623 +57 + dimension: 256 lr: 0.005 CNN channel: 32 auroc: 0.5318181818181819 f1: 0.7971014492753623 +58 + dimension: 256 lr: 0.005 CNN channel: 64 auroc: 0.588961038961039 f1: 0.7971014492753623 +59 + dimension: 256 lr: 0.001 CNN channel: 32 auroc: 0.5233766233766234 f1: 0.7971014492753623 +60 + dimension: 256 lr: 0.001 CNN channel: 64 auroc: 0.5038961038961038 f1: 0.7971014492753623 + +logprob with weight + +1 + dimension: 32 lr: 0.1 auroc: 0.5105820105820106 f1: 0.7575757575757576 +2 + dimension: 32 lr: 0.05 auroc: 0.4933862433862433 f1: 0.7642276422764228 +3 + dimension: 32 lr: 0.01 auroc: 0.4966931216931217 f1: 0.7619047619047619 +4 + dimension: 32 lr: 0.005 auroc: 0.5707671957671958 f1: 0.7538461538461538 +5 + dimension: 32 lr: 0.001 auroc: 0.5165343915343915 f1: 0.7580645161290323 +6 + dimension: 64 lr: 0.1 auroc: 0.5158730158730159 f1: 0.7642276422764228 +7 + dimension: 64 lr: 0.05 auroc: 0.5376984126984127 f1: 0.7580645161290323 +8 + dimension: 64 lr: 0.01 auroc: 0.4920634920634921 f1: 0.7580645161290323 +9 + dimension: 64 lr: 0.005 auroc: 0.4907407407407407 f1: 0.7619047619047619 +10 + dimension: 64 lr: 0.001 auroc: 0.6071428571428571 f1: 0.7666666666666667 +11 + dimension: 128 lr: 0.1 auroc: 0.4874338624338624 f1: 0.746031746031746 +12 + dimension: 128 lr: 0.05 auroc: 0.5026455026455027 f1: 0.7903225806451613 +13 + dimension: 128 lr: 0.01 auroc: 0.5892857142857143 f1: 0.7941176470588235 +14 + dimension: 128 lr: 0.005 auroc: 0.4867724867724868 f1: 0.7941176470588235 +15 + dimension: 128 lr: 0.001 auroc: 0.5383597883597884 f1: 0.8062015503875969 +16 + dimension: 256 lr: 0.1 auroc: 0.5562169312169312 f1: 0.75 +17 + dimension: 256 lr: 0.05 auroc: 0.5092592592592592 f1: 0.7394957983193278 +18 + dimension: 256 lr: 0.01 auroc: 0.5932539682539681 f1: 0.7851851851851852 +19 + dimension: 256 lr: 0.005 auroc: 0.5251322751322751 f1: 0.7619047619047619 +20 + dimension: 256 lr: 0.001 auroc: 0.4087301587301587 f1: 0.7941176470588235 +21 + dimension: 32 lr: 0.1 CNN channel: 32 auroc: 0.5945767195767195 f1: 0.75 +22 + dimension: 32 lr: 0.1 CNN channel: 64 auroc: 0.5383597883597884 f1: 0.7704918032786885 +23 + dimension: 32 lr: 0.05 CNN channel: 32 auroc: 0.5158730158730159 f1: 0.7540983606557377 +24 + dimension: 32 lr: 0.05 CNN channel: 64 auroc: 0.5132275132275133 f1: 0.7642276422764228 +25 + dimension: 32 lr: 0.01 CNN channel: 32 auroc: 0.55489417989418 f1: 0.78125 +26 + dimension: 32 lr: 0.01 CNN channel: 64 auroc: 0.47685185185185186 f1: 0.7596899224806202 +27 + dimension: 32 lr: 0.005 CNN channel: 32 auroc: 0.4669312169312169 f1: 0.7692307692307693 +28 + dimension: 32 lr: 0.005 CNN channel: 64 auroc: 0.5178571428571429 f1: 0.7741935483870968 +29 + dimension: 32 lr: 0.001 CNN channel: 32 auroc: 0.5833333333333334 f1: 0.7704918032786885 +30 + dimension: 32 lr: 0.001 CNN channel: 64 auroc: 0.6342592592592593 f1: 0.7768595041322314 +31 + dimension: 64 lr: 0.1 CNN channel: 32 auroc: 0.5099206349206349 f1: 0.7642276422764228 +32 + dimension: 64 lr: 0.1 CNN channel: 64 auroc: 0.5621693121693122 f1: 0.743801652892562 +33 + dimension: 64 lr: 0.05 CNN channel: 32 auroc: 0.55489417989418 f1: 0.7559055118110236 +34 + dimension: 64 lr: 0.05 CNN channel: 64 auroc: 0.5383597883597884 f1: 0.7642276422764228 +35 + dimension: 64 lr: 0.01 CNN channel: 32 auroc: 0.5112433862433863 f1: 0.768 +36 + dimension: 64 lr: 0.01 CNN channel: 64 auroc: 0.5297619047619048 f1: 0.7559055118110236 +37 + dimension: 64 lr: 0.005 CNN channel: 32 auroc: 0.576058201058201 f1: 0.7479674796747967 +38 + dimension: 64 lr: 0.005 CNN channel: 64 auroc: 0.6018518518518519 f1: 0.7580645161290323 +39 + dimension: 64 lr: 0.001 CNN channel: 32 auroc: 0.5582010582010581 f1: 0.6851851851851852 +40 + dimension: 64 lr: 0.001 CNN channel: 64 auroc: 0.3835978835978836 f1: 0.7941176470588235 +41 + dimension: 128 lr: 0.1 CNN channel: 32 auroc: 0.4973544973544973 f1: 0.7642276422764228 +42 + dimension: 128 lr: 0.1 CNN channel: 64 auroc: 0.5423280423280423 f1: 0.743801652892562 +43 + dimension: 128 lr: 0.05 CNN channel: 32 auroc: 0.5257936507936508 f1: 0.7580645161290323 +44 + dimension: 128 lr: 0.05 CNN channel: 64 auroc: 0.49206349206349204 f1: 0.7642276422764228 +45 + dimension: 128 lr: 0.01 CNN channel: 32 auroc: 0.6078042328042328 f1: 0.7666666666666667 +46 + dimension: 128 lr: 0.01 CNN channel: 64 auroc: 0.509920634920635 f1: 0.7580645161290323 +47 + dimension: 128 lr: 0.005 CNN channel: 32 auroc: 0.5436507936507936 f1: 0.7642276422764228 +48 + dimension: 128 lr: 0.005 CNN channel: 64 auroc: 0.531084656084656 f1: 0.8091603053435115 +49 + dimension: 128 lr: 0.001 CNN channel: 32 auroc: 0.6507936507936508 f1: 0.7804878048780488 +50 + dimension: 128 lr: 0.001 CNN channel: 64 auroc: 0.4444444444444444 f1: 0.7941176470588235 +51 + dimension: 256 lr: 0.1 CNN channel: 32 auroc: 0.5059523809523809 f1: 0.7642276422764228 +52 + dimension: 256 lr: 0.1 CNN channel: 64 auroc: 0.5707671957671958 f1: 0.7538461538461538 +53 + dimension: 256 lr: 0.05 CNN channel: 32 auroc: 0.49074074074074076 f1: 0.7642276422764228 +54 + dimension: 256 lr: 0.05 CNN channel: 64 auroc: 0.5205026455026455 f1: 0.7580645161290323 +55 + dimension: 256 lr: 0.01 CNN channel: 32 auroc: 0.5615079365079365 f1: 0.7580645161290323 +56 + dimension: 256 lr: 0.01 CNN channel: 64 auroc: 0.494047619047619 f1: 0.7540983606557377 +57 + dimension: 256 lr: 0.005 CNN channel: 32 auroc: 0.5542328042328042 f1: 0.7642276422764228 +58 + dimension: 256 lr: 0.005 CNN channel: 64 auroc: 0.6011904761904762 f1: 0.7941176470588235 +59 + dimension: 256 lr: 0.001 CNN channel: 32 auroc: 0.45436507936507936 f1: 0.7941176470588235 +60 + dimension: 256 lr: 0.001 CNN channel: 64 auroc: 0.5191798941798942 f1: 0.752 +1 + dimension: 32 lr: 0.1 auroc: 0.511904761904762 f1: 0.4523809523809524 +2 + dimension: 32 lr: 0.05 auroc: 0.5317460317460317 f1: 0.7619047619047619 +3 + dimension: 32 lr: 0.01 auroc: 0.47883597883597884 f1: 0.0 +4 + dimension: 32 lr: 0.005 auroc: 0.5046296296296297 f1: 0.7941176470588235 +5 + dimension: 32 lr: 0.001 auroc: 0.5363756613756614 f1: 0.7941176470588235 +6 + dimension: 64 lr: 0.1 auroc: 0.5337301587301587 f1: 0.7761194029850746 +7 + dimension: 64 lr: 0.05 auroc: 0.4986772486772486 f1: 0.6846846846846847 +8 + dimension: 64 lr: 0.01 auroc: 0.5337301587301587 f1: 0.7142857142857143 +9 + dimension: 64 lr: 0.005 auroc: 0.5654761904761905 f1: 0.7941176470588235 +10 + dimension: 64 lr: 0.001 auroc: 0.47619047619047616 f1: 0.7941176470588235 +11 + dimension: 128 lr: 0.1 auroc: 0.5178571428571429 f1: 0.7941176470588235 +12 + dimension: 128 lr: 0.05 auroc: 0.5562169312169312 f1: 0.7941176470588235 +13 + dimension: 128 lr: 0.01 auroc: 0.5059523809523809 f1: 0.7941176470588235 +14 + dimension: 128 lr: 0.005 auroc: 0.5515873015873016 f1: 0.7941176470588235 +15 + dimension: 128 lr: 0.001 auroc: 0.48809523809523814 f1: 0.0 +16 + dimension: 256 lr: 0.1 auroc: 0.5099206349206349 f1: 0.7941176470588235 +17 + dimension: 256 lr: 0.05 auroc: 0.5251322751322751 f1: 0.7941176470588235 +18 + dimension: 256 lr: 0.01 auroc: 0.5198412698412699 f1: 0.03636363636363636 +19 + dimension: 256 lr: 0.005 auroc: 0.5185185185185185 f1: 0.7941176470588235 +20 + dimension: 256 lr: 0.001 auroc: 0.4880952380952381 f1: 0.7642276422764228 +21 + dimension: 32 lr: 0.1 CNN channel: 32 auroc: 0.5575396825396826 f1: 0.7941176470588235 +22 + dimension: 32 lr: 0.1 CNN channel: 64 auroc: 0.5284391534391534 f1: 0.7603305785123967 +23 + dimension: 32 lr: 0.05 CNN channel: 32 auroc: 0.4887566137566137 f1: 0.7941176470588235 +24 + dimension: 32 lr: 0.05 CNN channel: 64 auroc: 0.494047619047619 f1: 0.7941176470588235 +25 + dimension: 32 lr: 0.01 CNN channel: 32 auroc: 0.5376984126984128 f1: 0.7941176470588235 +26 + dimension: 32 lr: 0.01 CNN channel: 64 auroc: 0.5310846560846562 f1: 0.7207207207207207 +27 + dimension: 32 lr: 0.005 CNN channel: 32 auroc: 0.5542328042328042 f1: 0.7941176470588235 +28 + dimension: 32 lr: 0.005 CNN channel: 64 auroc: 0.49007936507936506 f1: 0.7480916030534351 +29 + dimension: 32 lr: 0.001 CNN channel: 32 auroc: 0.4384920634920635 f1: 0.7941176470588235 +30 + dimension: 32 lr: 0.001 CNN channel: 64 auroc: 0.5787037037037037 f1: 0.7941176470588235 +31 + dimension: 64 lr: 0.1 CNN channel: 32 auroc: 0.5396825396825397 f1: 0.7941176470588235 +32 + dimension: 64 lr: 0.1 CNN channel: 64 auroc: 0.498015873015873 f1: 0.7559055118110236 +33 + dimension: 64 lr: 0.05 CNN channel: 32 auroc: 0.503968253968254 f1: 0.2857142857142857 +34 + dimension: 64 lr: 0.05 CNN channel: 64 auroc: 0.48743386243386244 f1: 0.7642276422764228 +35 + dimension: 64 lr: 0.01 CNN channel: 32 auroc: 0.5396825396825397 f1: 0.7941176470588235 +36 + dimension: 64 lr: 0.01 CNN channel: 64 auroc: 0.533068783068783 f1: 0.0 +37 + dimension: 64 lr: 0.005 CNN channel: 32 auroc: 0.498015873015873 f1: 0.7941176470588235 +38 + dimension: 64 lr: 0.005 CNN channel: 64 auroc: 0.49867724867724866 f1: 0.7941176470588235 +39 + dimension: 64 lr: 0.001 CNN channel: 32 auroc: 0.6190476190476191 f1: 0.7941176470588235 \ No newline at end of file diff --git a/models/linear/1.pth b/models/linear/1.pth index 37bb028..0603229 100644 Binary files a/models/linear/1.pth and b/models/linear/1.pth differ diff --git a/models/linear/10.pth b/models/linear/10.pth new file mode 100644 index 0000000..2c290cd Binary files /dev/null and b/models/linear/10.pth differ diff --git a/models/linear/11.pth b/models/linear/11.pth new file mode 100644 index 0000000..e15d9c2 Binary files /dev/null and b/models/linear/11.pth differ diff --git a/models/linear/12.pth b/models/linear/12.pth new file mode 100644 index 0000000..03e33c9 Binary files /dev/null and b/models/linear/12.pth differ diff --git a/models/linear/14.pth b/models/linear/14.pth new file mode 100644 index 0000000..28f6009 Binary files /dev/null and b/models/linear/14.pth differ diff --git a/models/linear/15.pth b/models/linear/15.pth new file mode 100644 index 0000000..2e9093a Binary files /dev/null and b/models/linear/15.pth differ diff --git a/models/linear/16.pth b/models/linear/16.pth new file mode 100644 index 0000000..eebce02 Binary files /dev/null and b/models/linear/16.pth differ diff --git a/models/linear/18.pth b/models/linear/18.pth new file mode 100644 index 0000000..efcc0db Binary files /dev/null and b/models/linear/18.pth differ diff --git a/models/linear/2.pth b/models/linear/2.pth index 5ca2c8a..0bcce55 100644 Binary files a/models/linear/2.pth and b/models/linear/2.pth differ diff --git a/models/linear/20.pth b/models/linear/20.pth new file mode 100644 index 0000000..79c11fe Binary files /dev/null and b/models/linear/20.pth differ diff --git a/models/linear/3.pth b/models/linear/3.pth index 49d389b..07319ad 100644 Binary files a/models/linear/3.pth and b/models/linear/3.pth differ diff --git a/models/linear/4.pth b/models/linear/4.pth index f764a3e..597ff27 100644 Binary files a/models/linear/4.pth and b/models/linear/4.pth differ diff --git a/models/linear/5.pth b/models/linear/5.pth new file mode 100644 index 0000000..1ea69e6 Binary files /dev/null and b/models/linear/5.pth differ diff --git a/models/linear/6.pth b/models/linear/6.pth new file mode 100644 index 0000000..497289c Binary files /dev/null and b/models/linear/6.pth differ diff --git a/models/linear/7.pth b/models/linear/7.pth new file mode 100644 index 0000000..41f183b Binary files /dev/null and b/models/linear/7.pth differ diff --git a/models/linear/8.pth b/models/linear/8.pth new file mode 100644 index 0000000..fd749b9 Binary files /dev/null and b/models/linear/8.pth differ diff --git a/models/linear/9.pth b/models/linear/9.pth new file mode 100644 index 0000000..40d8c0d Binary files /dev/null and b/models/linear/9.pth differ diff --git a/models/linear/weighted1.pth b/models/linear/weighted1.pth new file mode 100644 index 0000000..f5f1b24 Binary files /dev/null and b/models/linear/weighted1.pth differ diff --git a/models/linear/weighted10.pth b/models/linear/weighted10.pth new file mode 100644 index 0000000..e1a30a2 Binary files /dev/null and b/models/linear/weighted10.pth differ diff --git a/models/linear/weighted11.pth b/models/linear/weighted11.pth new file mode 100644 index 0000000..4bb66c9 Binary files /dev/null and b/models/linear/weighted11.pth differ diff --git a/models/linear/weighted12.pth b/models/linear/weighted12.pth new file mode 100644 index 0000000..61dc174 Binary files /dev/null and b/models/linear/weighted12.pth differ diff --git a/models/linear/weighted13.pth b/models/linear/weighted13.pth new file mode 100644 index 0000000..e72dfda Binary files /dev/null and b/models/linear/weighted13.pth differ diff --git a/models/linear/weighted14.pth b/models/linear/weighted14.pth new file mode 100644 index 0000000..a7c9858 Binary files /dev/null and b/models/linear/weighted14.pth differ diff --git a/models/linear/weighted15.pth b/models/linear/weighted15.pth new file mode 100644 index 0000000..0dd6b21 Binary files /dev/null and b/models/linear/weighted15.pth differ diff --git a/models/linear/weighted16.pth b/models/linear/weighted16.pth new file mode 100644 index 0000000..6465313 Binary files /dev/null and b/models/linear/weighted16.pth differ diff --git a/models/linear/weighted17.pth b/models/linear/weighted17.pth new file mode 100644 index 0000000..f158598 Binary files /dev/null and b/models/linear/weighted17.pth differ diff --git a/models/linear/weighted19.pth b/models/linear/weighted19.pth new file mode 100644 index 0000000..0b7a909 Binary files /dev/null and b/models/linear/weighted19.pth differ diff --git a/models/linear/weighted2.pth b/models/linear/weighted2.pth new file mode 100644 index 0000000..cd67bae Binary files /dev/null and b/models/linear/weighted2.pth differ diff --git a/models/linear/weighted20.pth b/models/linear/weighted20.pth new file mode 100644 index 0000000..a175d82 Binary files /dev/null and b/models/linear/weighted20.pth differ diff --git a/models/linear/weighted3.pth b/models/linear/weighted3.pth new file mode 100644 index 0000000..21cc304 Binary files /dev/null and b/models/linear/weighted3.pth differ diff --git a/models/linear/weighted4.pth b/models/linear/weighted4.pth new file mode 100644 index 0000000..8a3b9f8 Binary files /dev/null and b/models/linear/weighted4.pth differ diff --git a/models/linear/weighted5.pth b/models/linear/weighted5.pth new file mode 100644 index 0000000..75b62bb Binary files /dev/null and b/models/linear/weighted5.pth differ diff --git a/models/linear/weighted6.pth b/models/linear/weighted6.pth new file mode 100644 index 0000000..9ccb391 Binary files /dev/null and b/models/linear/weighted6.pth differ diff --git a/models/linear/weighted7.pth b/models/linear/weighted7.pth new file mode 100644 index 0000000..f6a50bd Binary files /dev/null and b/models/linear/weighted7.pth differ diff --git a/models/linear/weighted8.pth b/models/linear/weighted8.pth new file mode 100644 index 0000000..6f2e20f Binary files /dev/null and b/models/linear/weighted8.pth differ diff --git a/models/linear/weighted9.pth b/models/linear/weighted9.pth new file mode 100644 index 0000000..12f11f3 Binary files /dev/null and b/models/linear/weighted9.pth differ diff --git a/models/logprob/1.pth b/models/logprob/1.pth index 36d8551..eeedb4b 100644 Binary files a/models/logprob/1.pth and b/models/logprob/1.pth differ diff --git a/models/logprob/14.pth b/models/logprob/14.pth new file mode 100644 index 0000000..4522e6f Binary files /dev/null and b/models/logprob/14.pth differ diff --git a/models/logprob/2.pth b/models/logprob/2.pth deleted file mode 100644 index 4a40ef0..0000000 Binary files a/models/logprob/2.pth and /dev/null differ diff --git a/models/logprob/3.pth b/models/logprob/3.pth deleted file mode 100644 index dc50c16..0000000 Binary files a/models/logprob/3.pth and /dev/null differ diff --git a/models/logprob/39.pth b/models/logprob/39.pth new file mode 100644 index 0000000..20d03cc Binary files /dev/null and b/models/logprob/39.pth differ diff --git a/models/logprob/4.pth b/models/logprob/4.pth deleted file mode 100644 index 464e4ba..0000000 Binary files a/models/logprob/4.pth and /dev/null differ diff --git a/models/logprob/5.pth b/models/logprob/5.pth deleted file mode 100644 index 185e456..0000000 Binary files a/models/logprob/5.pth and /dev/null differ diff --git a/models/logprob/6.pth b/models/logprob/6.pth deleted file mode 100644 index c851135..0000000 Binary files a/models/logprob/6.pth and /dev/null differ diff --git a/models/logprob/7.pth b/models/logprob/7.pth deleted file mode 100644 index 21440b6..0000000 Binary files a/models/logprob/7.pth and /dev/null differ diff --git a/models/logprob/weighted10.pth b/models/logprob/weighted10.pth new file mode 100644 index 0000000..fdcba36 Binary files /dev/null and b/models/logprob/weighted10.pth differ diff --git a/models/logprob/weighted30.pth b/models/logprob/weighted30.pth new file mode 100644 index 0000000..95e8768 Binary files /dev/null and b/models/logprob/weighted30.pth differ diff --git a/models/logprob/weighted38.pth b/models/logprob/weighted38.pth new file mode 100644 index 0000000..ac16a27 Binary files /dev/null and b/models/logprob/weighted38.pth differ diff --git a/models/logprob/weighted45.pth b/models/logprob/weighted45.pth new file mode 100644 index 0000000..4313e5d Binary files /dev/null and b/models/logprob/weighted45.pth differ diff --git a/models/logprob/weighted49.pth b/models/logprob/weighted49.pth new file mode 100644 index 0000000..929b3a0 Binary files /dev/null and b/models/logprob/weighted49.pth differ diff --git a/models/logprob/weighted58.pth b/models/logprob/weighted58.pth new file mode 100644 index 0000000..bbfbfd7 Binary files /dev/null and b/models/logprob/weighted58.pth differ diff --git a/nice_table.png b/nice_table.png new file mode 100644 index 0000000..d7f34db Binary files /dev/null and b/nice_table.png differ diff --git a/test_results/ROC/CNN comparison.png b/test_results/ROC/CNN comparison.png new file mode 100644 index 0000000..ca71860 Binary files /dev/null and b/test_results/ROC/CNN comparison.png differ diff --git a/test_results/ROC/MLP #16 hybrid #1 #39 noweight.png b/test_results/ROC/MLP #16 hybrid #1 #39 noweight.png new file mode 100644 index 0000000..6c4a16a Binary files /dev/null and b/test_results/ROC/MLP #16 hybrid #1 #39 noweight.png differ diff --git a/test_results/ROC/MLP #16 hybrid #1 comparison.png b/test_results/ROC/MLP #16 hybrid #1 comparison.png new file mode 100644 index 0000000..44c7d15 Binary files /dev/null and b/test_results/ROC/MLP #16 hybrid #1 comparison.png differ diff --git a/test_results/ROC/hybrid linear comparison.png b/test_results/ROC/hybrid linear comparison.png new file mode 100644 index 0000000..3d204d2 Binary files /dev/null and b/test_results/ROC/hybrid linear comparison.png differ diff --git a/test_results/ROC/linear/1.pdf b/test_results/ROC/linear/1.pdf deleted file mode 100644 index b5fd602..0000000 Binary files a/test_results/ROC/linear/1.pdf and /dev/null differ diff --git a/test_results/ROC/linear/4.pdf b/test_results/ROC/linear/15.pdf similarity index 80% rename from test_results/ROC/linear/4.pdf rename to test_results/ROC/linear/15.pdf index d6b6305..8cf3f29 100644 Binary files a/test_results/ROC/linear/4.pdf and b/test_results/ROC/linear/15.pdf differ diff --git a/test_results/ROC/linear/2.pdf b/test_results/ROC/linear/2.pdf deleted file mode 100644 index a1fdf55..0000000 Binary files a/test_results/ROC/linear/2.pdf and /dev/null differ diff --git a/test_results/ROC/linear/3.pdf b/test_results/ROC/linear/3.pdf deleted file mode 100644 index bcccb15..0000000 Binary files a/test_results/ROC/linear/3.pdf and /dev/null differ diff --git a/test_results/ROC/linear/linear model weight comparison.png b/test_results/ROC/linear/linear model weight comparison.png new file mode 100644 index 0000000..2731428 Binary files /dev/null and b/test_results/ROC/linear/linear model weight comparison.png differ diff --git a/test_results/ROC/logprob/1.pdf b/test_results/ROC/logprob/1.pdf index eb8a3aa..e42220f 100644 Binary files a/test_results/ROC/logprob/1.pdf and b/test_results/ROC/logprob/1.pdf differ diff --git a/test_results/ROC/logprob/5.pdf b/test_results/ROC/logprob/14.pdf similarity index 80% rename from test_results/ROC/logprob/5.pdf rename to test_results/ROC/logprob/14.pdf index e13784b..d424d2c 100644 Binary files a/test_results/ROC/logprob/5.pdf and b/test_results/ROC/logprob/14.pdf differ diff --git a/test_results/ROC/logprob/2.pdf b/test_results/ROC/logprob/2.pdf deleted file mode 100644 index 8fe1b6e..0000000 Binary files a/test_results/ROC/logprob/2.pdf and /dev/null differ diff --git a/test_results/ROC/logprob/3.pdf b/test_results/ROC/logprob/3.pdf deleted file mode 100644 index 7b4e42e..0000000 Binary files a/test_results/ROC/logprob/3.pdf and /dev/null differ diff --git a/test_results/ROC/logprob/7.pdf b/test_results/ROC/logprob/39.pdf similarity index 80% rename from test_results/ROC/logprob/7.pdf rename to test_results/ROC/logprob/39.pdf index 31183bc..c79cfc1 100644 Binary files a/test_results/ROC/logprob/7.pdf and b/test_results/ROC/logprob/39.pdf differ diff --git a/test_results/ROC/logprob/4.pdf b/test_results/ROC/logprob/4.pdf deleted file mode 100644 index 12d73f3..0000000 Binary files a/test_results/ROC/logprob/4.pdf and /dev/null differ diff --git a/test_results/ROC/logprob/6.pdf b/test_results/ROC/logprob/6.pdf deleted file mode 100644 index 7bd3ede..0000000 Binary files a/test_results/ROC/logprob/6.pdf and /dev/null differ diff --git a/test_results/ROC/logprob/8.pdf b/test_results/ROC/logprob/8.pdf deleted file mode 100644 index 671ee18..0000000 Binary files a/test_results/ROC/logprob/8.pdf and /dev/null differ diff --git a/test_results/ROC/logprob/logprob model CNN weight comparison.png b/test_results/ROC/logprob/logprob model CNN weight comparison.png new file mode 100644 index 0000000..01ed6c3 Binary files /dev/null and b/test_results/ROC/logprob/logprob model CNN weight comparison.png differ diff --git a/test_results/ROC/weighted hybrid #10 #49 MLP #15.png b/test_results/ROC/weighted hybrid #10 #49 MLP #15.png new file mode 100644 index 0000000..11974e3 Binary files /dev/null and b/test_results/ROC/weighted hybrid #10 #49 MLP #15.png differ diff --git a/test_results/ROC/weighted hybrid vs MLP .png b/test_results/ROC/weighted hybrid vs MLP .png new file mode 100644 index 0000000..3d69995 Binary files /dev/null and b/test_results/ROC/weighted hybrid vs MLP .png differ diff --git a/test_results/distributions/0.5 threshold linear 3.png b/test_results/distributions/0.5 threshold linear 3.png new file mode 100644 index 0000000..ba0c6fb Binary files /dev/null and b/test_results/distributions/0.5 threshold linear 3.png differ diff --git a/test_results/distributions/0.5 threshold linear distribution.png b/test_results/distributions/0.5 threshold linear distribution.png new file mode 100644 index 0000000..a45bb71 Binary files /dev/null and b/test_results/distributions/0.5 threshold linear distribution.png differ diff --git a/test_results/distributions/Figure_2.png b/test_results/distributions/Figure_2.png new file mode 100644 index 0000000..d6b506e Binary files /dev/null and b/test_results/distributions/Figure_2.png differ diff --git a/test_results/distributions/MLP #16 non-weighted.png b/test_results/distributions/MLP #16 non-weighted.png new file mode 100644 index 0000000..394cbfc Binary files /dev/null and b/test_results/distributions/MLP #16 non-weighted.png differ diff --git a/test_results/distributions/MLP not-weighted1.png b/test_results/distributions/MLP not-weighted1.png new file mode 100644 index 0000000..bc0565f Binary files /dev/null and b/test_results/distributions/MLP not-weighted1.png differ diff --git a/test_results/distributions/hybrid #1 without weight.png b/test_results/distributions/hybrid #1 without weight.png new file mode 100644 index 0000000..8beee0a Binary files /dev/null and b/test_results/distributions/hybrid #1 without weight.png differ diff --git a/test_results/distributions/hybrid #14 without weight.png b/test_results/distributions/hybrid #14 without weight.png new file mode 100644 index 0000000..190882c Binary files /dev/null and b/test_results/distributions/hybrid #14 without weight.png differ diff --git a/test_results/distributions/hybrid #39 not weighted.png b/test_results/distributions/hybrid #39 not weighted.png new file mode 100644 index 0000000..eb1ec83 Binary files /dev/null and b/test_results/distributions/hybrid #39 not weighted.png differ diff --git a/test_results/distributions/hybrid output distribution.png b/test_results/distributions/hybrid output distribution.png new file mode 100644 index 0000000..c1fdbb5 Binary files /dev/null and b/test_results/distributions/hybrid output distribution.png differ diff --git a/test_results/distributions/hybrid output without CNN distribution.png b/test_results/distributions/hybrid output without CNN distribution.png new file mode 100644 index 0000000..968e7d2 Binary files /dev/null and b/test_results/distributions/hybrid output without CNN distribution.png differ diff --git a/test_results/distributions/linear distribution both threshold extreme.png b/test_results/distributions/linear distribution both threshold extreme.png new file mode 100644 index 0000000..03577af Binary files /dev/null and b/test_results/distributions/linear distribution both threshold extreme.png differ diff --git a/test_results/distributions/linear model distribution with both thresholds.png b/test_results/distributions/linear model distribution with both thresholds.png new file mode 100644 index 0000000..15c3ebd Binary files /dev/null and b/test_results/distributions/linear model distribution with both thresholds.png differ diff --git a/test_results/distributions/original hybrid #10 weighted.png b/test_results/distributions/original hybrid #10 weighted.png new file mode 100644 index 0000000..186e559 Binary files /dev/null and b/test_results/distributions/original hybrid #10 weighted.png differ diff --git a/test_results/loss/linear/1.pdf b/test_results/loss/linear/1.pdf new file mode 100644 index 0000000..3c976f6 Binary files /dev/null and b/test_results/loss/linear/1.pdf differ diff --git a/test_results/loss/linear/10.pdf b/test_results/loss/linear/10.pdf new file mode 100644 index 0000000..b583296 Binary files /dev/null and b/test_results/loss/linear/10.pdf differ diff --git a/test_results/loss/linear/11.pdf b/test_results/loss/linear/11.pdf new file mode 100644 index 0000000..b5ccff7 Binary files /dev/null and b/test_results/loss/linear/11.pdf differ diff --git a/test_results/loss/linear/12.pdf b/test_results/loss/linear/12.pdf new file mode 100644 index 0000000..8097b84 Binary files /dev/null and b/test_results/loss/linear/12.pdf differ diff --git a/test_results/loss/linear/14.pdf b/test_results/loss/linear/14.pdf new file mode 100644 index 0000000..ca01963 Binary files /dev/null and b/test_results/loss/linear/14.pdf differ diff --git a/test_results/loss/linear/15.pdf b/test_results/loss/linear/15.pdf new file mode 100644 index 0000000..8a10f4b Binary files /dev/null and b/test_results/loss/linear/15.pdf differ diff --git a/test_results/loss/linear/16.pdf b/test_results/loss/linear/16.pdf new file mode 100644 index 0000000..c113d79 Binary files /dev/null and b/test_results/loss/linear/16.pdf differ diff --git a/test_results/loss/linear/18.pdf b/test_results/loss/linear/18.pdf new file mode 100644 index 0000000..90cd7e4 Binary files /dev/null and b/test_results/loss/linear/18.pdf differ diff --git a/test_results/loss/linear/2.pdf b/test_results/loss/linear/2.pdf new file mode 100644 index 0000000..128829e Binary files /dev/null and b/test_results/loss/linear/2.pdf differ diff --git a/test_results/loss/linear/20.pdf b/test_results/loss/linear/20.pdf new file mode 100644 index 0000000..81a4e6b Binary files /dev/null and b/test_results/loss/linear/20.pdf differ diff --git a/test_results/loss/linear/3.pdf b/test_results/loss/linear/3.pdf new file mode 100644 index 0000000..12da14c Binary files /dev/null and b/test_results/loss/linear/3.pdf differ diff --git a/test_results/loss/linear/4.pdf b/test_results/loss/linear/4.pdf new file mode 100644 index 0000000..397914d Binary files /dev/null and b/test_results/loss/linear/4.pdf differ diff --git a/test_results/loss/linear/5.pdf b/test_results/loss/linear/5.pdf new file mode 100644 index 0000000..760bb07 Binary files /dev/null and b/test_results/loss/linear/5.pdf differ diff --git a/test_results/loss/linear/6.pdf b/test_results/loss/linear/6.pdf new file mode 100644 index 0000000..497849c Binary files /dev/null and b/test_results/loss/linear/6.pdf differ diff --git a/test_results/loss/linear/7.pdf b/test_results/loss/linear/7.pdf new file mode 100644 index 0000000..b6e89ea Binary files /dev/null and b/test_results/loss/linear/7.pdf differ diff --git a/test_results/loss/linear/8.pdf b/test_results/loss/linear/8.pdf new file mode 100644 index 0000000..dc89059 Binary files /dev/null and b/test_results/loss/linear/8.pdf differ diff --git a/test_results/loss/linear/9.pdf b/test_results/loss/linear/9.pdf new file mode 100644 index 0000000..1a4a615 Binary files /dev/null and b/test_results/loss/linear/9.pdf differ diff --git a/test_results/loss/linear/linear_1.pdf b/test_results/loss/linear/linear/linear_1.pdf similarity index 80% rename from test_results/loss/linear/linear_1.pdf rename to test_results/loss/linear/linear/linear_1.pdf index 40374cd..114daff 100644 Binary files a/test_results/loss/linear/linear_1.pdf and b/test_results/loss/linear/linear/linear_1.pdf differ diff --git a/test_results/loss/linear/linear/linear_10.pdf b/test_results/loss/linear/linear/linear_10.pdf new file mode 100644 index 0000000..173f6e4 Binary files /dev/null and b/test_results/loss/linear/linear/linear_10.pdf differ diff --git a/test_results/loss/linear/linear/linear_11.pdf b/test_results/loss/linear/linear/linear_11.pdf new file mode 100644 index 0000000..a3dc05d Binary files /dev/null and b/test_results/loss/linear/linear/linear_11.pdf differ diff --git a/test_results/loss/linear/linear/linear_12.pdf b/test_results/loss/linear/linear/linear_12.pdf new file mode 100644 index 0000000..275ec93 Binary files /dev/null and b/test_results/loss/linear/linear/linear_12.pdf differ diff --git a/test_results/loss/linear/linear/linear_13.pdf b/test_results/loss/linear/linear/linear_13.pdf new file mode 100644 index 0000000..ea1ac30 Binary files /dev/null and b/test_results/loss/linear/linear/linear_13.pdf differ diff --git a/test_results/loss/linear/linear/linear_14.pdf b/test_results/loss/linear/linear/linear_14.pdf new file mode 100644 index 0000000..7396882 Binary files /dev/null and b/test_results/loss/linear/linear/linear_14.pdf differ diff --git a/test_results/loss/linear/linear/linear_15.pdf b/test_results/loss/linear/linear/linear_15.pdf new file mode 100644 index 0000000..428a701 Binary files /dev/null and b/test_results/loss/linear/linear/linear_15.pdf differ diff --git a/test_results/loss/linear/linear/linear_16.pdf b/test_results/loss/linear/linear/linear_16.pdf new file mode 100644 index 0000000..7877bc0 Binary files /dev/null and b/test_results/loss/linear/linear/linear_16.pdf differ diff --git a/test_results/loss/linear/linear/linear_17.pdf b/test_results/loss/linear/linear/linear_17.pdf new file mode 100644 index 0000000..06a8e4d Binary files /dev/null and b/test_results/loss/linear/linear/linear_17.pdf differ diff --git a/test_results/loss/linear/linear/linear_18.pdf b/test_results/loss/linear/linear/linear_18.pdf new file mode 100644 index 0000000..8695deb Binary files /dev/null and b/test_results/loss/linear/linear/linear_18.pdf differ diff --git a/test_results/loss/linear/linear/linear_19.pdf b/test_results/loss/linear/linear/linear_19.pdf new file mode 100644 index 0000000..a7ca58c Binary files /dev/null and b/test_results/loss/linear/linear/linear_19.pdf differ diff --git a/test_results/loss/linear/linear_2.pdf b/test_results/loss/linear/linear/linear_2.pdf similarity index 79% rename from test_results/loss/linear/linear_2.pdf rename to test_results/loss/linear/linear/linear_2.pdf index e0ce114..6954e71 100644 Binary files a/test_results/loss/linear/linear_2.pdf and b/test_results/loss/linear/linear/linear_2.pdf differ diff --git a/test_results/loss/linear/linear/linear_20.pdf b/test_results/loss/linear/linear/linear_20.pdf new file mode 100644 index 0000000..fca6e64 Binary files /dev/null and b/test_results/loss/linear/linear/linear_20.pdf differ diff --git a/test_results/loss/linear/linear_3.pdf b/test_results/loss/linear/linear/linear_3.pdf similarity index 77% rename from test_results/loss/linear/linear_3.pdf rename to test_results/loss/linear/linear/linear_3.pdf index 8f478e3..9fbef95 100644 Binary files a/test_results/loss/linear/linear_3.pdf and b/test_results/loss/linear/linear/linear_3.pdf differ diff --git a/test_results/loss/linear/linear/linear_4.pdf b/test_results/loss/linear/linear/linear_4.pdf new file mode 100644 index 0000000..8cfea6c Binary files /dev/null and b/test_results/loss/linear/linear/linear_4.pdf differ diff --git a/test_results/loss/linear/linear/linear_5.pdf b/test_results/loss/linear/linear/linear_5.pdf new file mode 100644 index 0000000..74bae30 Binary files /dev/null and b/test_results/loss/linear/linear/linear_5.pdf differ diff --git a/test_results/loss/linear/linear/linear_6.pdf b/test_results/loss/linear/linear/linear_6.pdf new file mode 100644 index 0000000..028c4c0 Binary files /dev/null and b/test_results/loss/linear/linear/linear_6.pdf differ diff --git a/test_results/loss/linear/linear/linear_7.pdf b/test_results/loss/linear/linear/linear_7.pdf new file mode 100644 index 0000000..5e3dedd Binary files /dev/null and b/test_results/loss/linear/linear/linear_7.pdf differ diff --git a/test_results/loss/linear/linear/linear_8.pdf b/test_results/loss/linear/linear/linear_8.pdf new file mode 100644 index 0000000..a099ec4 Binary files /dev/null and b/test_results/loss/linear/linear/linear_8.pdf differ diff --git a/test_results/loss/linear/linear/linear_9.pdf b/test_results/loss/linear/linear/linear_9.pdf new file mode 100644 index 0000000..d6b2af5 Binary files /dev/null and b/test_results/loss/linear/linear/linear_9.pdf differ diff --git a/test_results/loss/linear/linear/weighted1.pdf b/test_results/loss/linear/linear/weighted1.pdf new file mode 100644 index 0000000..0818e24 Binary files /dev/null and b/test_results/loss/linear/linear/weighted1.pdf differ diff --git a/test_results/loss/linear/linear/weighted10.pdf b/test_results/loss/linear/linear/weighted10.pdf new file mode 100644 index 0000000..6336337 Binary files /dev/null and b/test_results/loss/linear/linear/weighted10.pdf differ diff --git a/test_results/loss/linear/linear/weighted11.pdf b/test_results/loss/linear/linear/weighted11.pdf new file mode 100644 index 0000000..df229cf Binary files /dev/null and b/test_results/loss/linear/linear/weighted11.pdf differ diff --git a/test_results/loss/linear/linear/weighted12.pdf b/test_results/loss/linear/linear/weighted12.pdf new file mode 100644 index 0000000..1ddcf68 Binary files /dev/null and b/test_results/loss/linear/linear/weighted12.pdf differ diff --git a/test_results/loss/linear/linear/weighted13.pdf b/test_results/loss/linear/linear/weighted13.pdf new file mode 100644 index 0000000..5faf426 Binary files /dev/null and b/test_results/loss/linear/linear/weighted13.pdf differ diff --git a/test_results/loss/linear/linear/weighted14.pdf b/test_results/loss/linear/linear/weighted14.pdf new file mode 100644 index 0000000..468d65c Binary files /dev/null and b/test_results/loss/linear/linear/weighted14.pdf differ diff --git a/test_results/loss/linear/linear/weighted15.pdf b/test_results/loss/linear/linear/weighted15.pdf new file mode 100644 index 0000000..a2dcfa2 Binary files /dev/null and b/test_results/loss/linear/linear/weighted15.pdf differ diff --git a/test_results/loss/linear/linear/weighted16.pdf b/test_results/loss/linear/linear/weighted16.pdf new file mode 100644 index 0000000..3c02a81 Binary files /dev/null and b/test_results/loss/linear/linear/weighted16.pdf differ diff --git a/test_results/loss/linear/linear/weighted17.pdf b/test_results/loss/linear/linear/weighted17.pdf new file mode 100644 index 0000000..58780f8 Binary files /dev/null and b/test_results/loss/linear/linear/weighted17.pdf differ diff --git a/test_results/loss/linear/linear/weighted18.pdf b/test_results/loss/linear/linear/weighted18.pdf new file mode 100644 index 0000000..b30f87e Binary files /dev/null and b/test_results/loss/linear/linear/weighted18.pdf differ diff --git a/test_results/loss/linear/linear/weighted19.pdf b/test_results/loss/linear/linear/weighted19.pdf new file mode 100644 index 0000000..c0aa6fc Binary files /dev/null and b/test_results/loss/linear/linear/weighted19.pdf differ diff --git a/test_results/loss/linear/linear/weighted2.pdf b/test_results/loss/linear/linear/weighted2.pdf new file mode 100644 index 0000000..c2aad5c Binary files /dev/null and b/test_results/loss/linear/linear/weighted2.pdf differ diff --git a/test_results/loss/linear/linear/weighted3.pdf b/test_results/loss/linear/linear/weighted3.pdf new file mode 100644 index 0000000..5939784 Binary files /dev/null and b/test_results/loss/linear/linear/weighted3.pdf differ diff --git a/test_results/loss/linear/linear/weighted4.pdf b/test_results/loss/linear/linear/weighted4.pdf new file mode 100644 index 0000000..a6a8b3c Binary files /dev/null and b/test_results/loss/linear/linear/weighted4.pdf differ diff --git a/test_results/loss/linear/linear/weighted5.pdf b/test_results/loss/linear/linear/weighted5.pdf new file mode 100644 index 0000000..bea0606 Binary files /dev/null and b/test_results/loss/linear/linear/weighted5.pdf differ diff --git a/test_results/loss/linear/linear/weighted6.pdf b/test_results/loss/linear/linear/weighted6.pdf new file mode 100644 index 0000000..511d620 Binary files /dev/null and b/test_results/loss/linear/linear/weighted6.pdf differ diff --git a/test_results/loss/linear/linear/weighted7.pdf b/test_results/loss/linear/linear/weighted7.pdf new file mode 100644 index 0000000..32f8453 Binary files /dev/null and b/test_results/loss/linear/linear/weighted7.pdf differ diff --git a/test_results/loss/linear/linear/weighted8.pdf b/test_results/loss/linear/linear/weighted8.pdf new file mode 100644 index 0000000..a795e09 Binary files /dev/null and b/test_results/loss/linear/linear/weighted8.pdf differ diff --git a/test_results/loss/linear/linear/weighted9.pdf b/test_results/loss/linear/linear/weighted9.pdf new file mode 100644 index 0000000..80dcfa9 Binary files /dev/null and b/test_results/loss/linear/linear/weighted9.pdf differ diff --git a/test_results/loss/linear/linear_weighted1.pdf b/test_results/loss/linear/linear_weighted1.pdf new file mode 100644 index 0000000..c93ff8c Binary files /dev/null and b/test_results/loss/linear/linear_weighted1.pdf differ diff --git a/test_results/loss/logprob/logprob_5.pdf b/test_results/loss/linear/linear_weighted10.pdf similarity index 79% rename from test_results/loss/logprob/logprob_5.pdf rename to test_results/loss/linear/linear_weighted10.pdf index 7e0653a..06ac9a7 100644 Binary files a/test_results/loss/logprob/logprob_5.pdf and b/test_results/loss/linear/linear_weighted10.pdf differ diff --git a/test_results/loss/linear/linear_weighted11.pdf b/test_results/loss/linear/linear_weighted11.pdf new file mode 100644 index 0000000..eb26464 Binary files /dev/null and b/test_results/loss/linear/linear_weighted11.pdf differ diff --git a/test_results/loss/logprob/logprob_4.pdf b/test_results/loss/linear/linear_weighted12.pdf similarity index 83% rename from test_results/loss/logprob/logprob_4.pdf rename to test_results/loss/linear/linear_weighted12.pdf index 49b4846..62e645b 100644 Binary files a/test_results/loss/logprob/logprob_4.pdf and b/test_results/loss/linear/linear_weighted12.pdf differ diff --git a/test_results/loss/linear/linear_weighted13.pdf b/test_results/loss/linear/linear_weighted13.pdf new file mode 100644 index 0000000..d12a8fc Binary files /dev/null and b/test_results/loss/linear/linear_weighted13.pdf differ diff --git a/test_results/loss/linear/linear_weighted14.pdf b/test_results/loss/linear/linear_weighted14.pdf new file mode 100644 index 0000000..11a04a9 Binary files /dev/null and b/test_results/loss/linear/linear_weighted14.pdf differ diff --git a/test_results/loss/linear/linear_weighted15.pdf b/test_results/loss/linear/linear_weighted15.pdf new file mode 100644 index 0000000..2a35ad8 Binary files /dev/null and b/test_results/loss/linear/linear_weighted15.pdf differ diff --git a/test_results/loss/linear/linear_weighted16.pdf b/test_results/loss/linear/linear_weighted16.pdf new file mode 100644 index 0000000..5517a2e Binary files /dev/null and b/test_results/loss/linear/linear_weighted16.pdf differ diff --git a/test_results/loss/linear/linear_weighted17.pdf b/test_results/loss/linear/linear_weighted17.pdf new file mode 100644 index 0000000..007f7f2 Binary files /dev/null and b/test_results/loss/linear/linear_weighted17.pdf differ diff --git a/test_results/loss/logprob/logprob_3.pdf b/test_results/loss/linear/linear_weighted19.pdf similarity index 79% rename from test_results/loss/logprob/logprob_3.pdf rename to test_results/loss/linear/linear_weighted19.pdf index 132469d..2305ebd 100644 Binary files a/test_results/loss/logprob/logprob_3.pdf and b/test_results/loss/linear/linear_weighted19.pdf differ diff --git a/test_results/loss/linear/linear_weighted2.pdf b/test_results/loss/linear/linear_weighted2.pdf new file mode 100644 index 0000000..56f672f Binary files /dev/null and b/test_results/loss/linear/linear_weighted2.pdf differ diff --git a/test_results/loss/linear/linear_weighted20.pdf b/test_results/loss/linear/linear_weighted20.pdf new file mode 100644 index 0000000..a75d0ab Binary files /dev/null and b/test_results/loss/linear/linear_weighted20.pdf differ diff --git a/test_results/loss/logprob/logprob_6.pdf b/test_results/loss/linear/linear_weighted3.pdf similarity index 78% rename from test_results/loss/logprob/logprob_6.pdf rename to test_results/loss/linear/linear_weighted3.pdf index af8fe3d..0e7e530 100644 Binary files a/test_results/loss/logprob/logprob_6.pdf and b/test_results/loss/linear/linear_weighted3.pdf differ diff --git a/test_results/loss/linear/linear_weighted4.pdf b/test_results/loss/linear/linear_weighted4.pdf new file mode 100644 index 0000000..dd38981 Binary files /dev/null and b/test_results/loss/linear/linear_weighted4.pdf differ diff --git a/test_results/loss/linear/linear_weighted5.pdf b/test_results/loss/linear/linear_weighted5.pdf new file mode 100644 index 0000000..a7c457b Binary files /dev/null and b/test_results/loss/linear/linear_weighted5.pdf differ diff --git a/test_results/loss/linear/linear_weighted6.pdf b/test_results/loss/linear/linear_weighted6.pdf new file mode 100644 index 0000000..448c901 Binary files /dev/null and b/test_results/loss/linear/linear_weighted6.pdf differ diff --git a/test_results/loss/linear/linear_weighted7.pdf b/test_results/loss/linear/linear_weighted7.pdf new file mode 100644 index 0000000..ccef080 Binary files /dev/null and b/test_results/loss/linear/linear_weighted7.pdf differ diff --git a/test_results/loss/linear/linear_weighted8.pdf b/test_results/loss/linear/linear_weighted8.pdf new file mode 100644 index 0000000..e81a9a8 Binary files /dev/null and b/test_results/loss/linear/linear_weighted8.pdf differ diff --git a/test_results/loss/linear/linear_weighted9.pdf b/test_results/loss/linear/linear_weighted9.pdf new file mode 100644 index 0000000..ea653d2 Binary files /dev/null and b/test_results/loss/linear/linear_weighted9.pdf differ diff --git a/test_results/loss/logprob/1.pdf b/test_results/loss/logprob/1.pdf new file mode 100644 index 0000000..c777238 Binary files /dev/null and b/test_results/loss/logprob/1.pdf differ diff --git a/test_results/loss/logprob/14.pdf b/test_results/loss/logprob/14.pdf new file mode 100644 index 0000000..27ef9a2 Binary files /dev/null and b/test_results/loss/logprob/14.pdf differ diff --git a/test_results/loss/logprob/39.pdf b/test_results/loss/logprob/39.pdf new file mode 100644 index 0000000..bac613d Binary files /dev/null and b/test_results/loss/logprob/39.pdf differ diff --git a/test_results/loss/logprob/logprob_2.pdf b/test_results/loss/logprob/weighted10.pdf similarity index 81% rename from test_results/loss/logprob/logprob_2.pdf rename to test_results/loss/logprob/weighted10.pdf index 7d07b57..4846711 100644 Binary files a/test_results/loss/logprob/logprob_2.pdf and b/test_results/loss/logprob/weighted10.pdf differ diff --git a/test_results/loss/logprob/weighted30.pdf b/test_results/loss/logprob/weighted30.pdf new file mode 100644 index 0000000..83756f5 Binary files /dev/null and b/test_results/loss/logprob/weighted30.pdf differ diff --git a/test_results/loss/logprob/weighted38.pdf b/test_results/loss/logprob/weighted38.pdf new file mode 100644 index 0000000..4deba88 Binary files /dev/null and b/test_results/loss/logprob/weighted38.pdf differ diff --git a/test_results/loss/logprob/weighted45.pdf b/test_results/loss/logprob/weighted45.pdf new file mode 100644 index 0000000..0d5f097 Binary files /dev/null and b/test_results/loss/logprob/weighted45.pdf differ diff --git a/test_results/loss/logprob/logprob_1.pdf b/test_results/loss/logprob/weighted49.pdf similarity index 79% rename from test_results/loss/logprob/logprob_1.pdf rename to test_results/loss/logprob/weighted49.pdf index 66d6315..cc3742f 100644 Binary files a/test_results/loss/logprob/logprob_1.pdf and b/test_results/loss/logprob/weighted49.pdf differ diff --git a/test_results/loss/logprob/weighted58.pdf b/test_results/loss/logprob/weighted58.pdf new file mode 100644 index 0000000..dc5c047 Binary files /dev/null and b/test_results/loss/logprob/weighted58.pdf differ