PHOBERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "PHOBERT.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7tw5qzU-Uh38"
      },
      "source": [
        "import os\n",
        "path = '/content/drive/MyDrive/VLSP_ReINTEL'\n",
        "#you should change this path to your project folder path\n",
        "os.chdir(path)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "73lM2My9KzRN",
        "outputId": "0f2d178d-73a9-4f81-b9a6-bb444e5f42ec"
      },
      "source": [
        "# install requirement library\n",
        "!pip install transformers\n",
        "!pip install fastBPE\n",
        "!pip install fairseq\n",
        "!pip install vncorenlp"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/84/7bc03215279f603125d844bf81c3fb3f2d50fe8e511546eb4897e4be2067/transformers-4.0.0-py3-none-any.whl (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 12.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 53.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 50.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.11.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=05c2b631061fd37c692d1ab0a0484d65334c8e46a3ea630b221b7a3c4791757d\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.0.0\n",
            "Collecting fastBPE\n",
            "  Downloading https://files.pythonhosted.org/packages/e1/37/f97181428a5d151501b90b2cebedf97c81b034ace753606a3cda5ad4e6e2/fastBPE-0.1.0.tar.gz\n",
            "Building wheels for collected packages: fastBPE\n",
            "  Building wheel for fastBPE (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fastBPE: filename=fastBPE-0.1.0-cp36-cp36m-linux_x86_64.whl size=481492 sha256=ce8695a31b1c5fb782a0f8873dfb5d9a75748ad2d22345dfe104aa071073acf7\n",
            "  Stored in directory: /root/.cache/pip/wheels/f3/0c/9c/fc62058b4d473a5602bcd3d3edfece796f123875379ea82d79\n",
            "Successfully built fastBPE\n",
            "Installing collected packages: fastBPE\n",
            "Successfully installed fastBPE-0.1.0\n",
            "Collecting fairseq\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/da/7c7032988dade3b21ccfd5b226e50b382abfd3459129d67240bb004506ae/fairseq-0.10.1-cp36-cp36m-manylinux1_x86_64.whl (1.7MB)\n",
            "\u001b[K     |████████████████████████████████| 1.7MB 12.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from fairseq) (1.14.3)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from fairseq) (0.8)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from fairseq) (2019.12.20)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (from fairseq) (0.29.21)\n",
            "Collecting sacrebleu>=1.4.12\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a3/c4/8e948f601a4f9609e8b2b58f31966cb13cf17b940b82aa3e767f01c42c52/sacrebleu-1.4.14-py3-none-any.whl (64kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 10.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from fairseq) (1.7.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from fairseq) (1.18.5)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from fairseq) (4.41.1)\n",
            "Collecting hydra-core\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f0/1f/7f502b9e37596164111655861370b08626f46f9e4524433c354f472765d4/hydra_core-1.0.4-py3-none-any.whl (122kB)\n",
            "\u001b[K     |████████████████████████████████| 122kB 63.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->fairseq) (2.20)\n",
            "Collecting portalocker\n",
            "  Downloading https://files.pythonhosted.org/packages/89/a6/3814b7107e0788040870e8825eebf214d72166adf656ba7d4bf14759a06a/portalocker-2.0.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->fairseq) (0.16.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch->fairseq) (3.7.4.3)\n",
            "Collecting antlr4-python3-runtime==4.8\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/02/789a0bddf9c9b31b14c3e79ec22b9656185a803dc31c15f006f9855ece0d/antlr4-python3-runtime-4.8.tar.gz (112kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 57.4MB/s \n",
            "\u001b[?25hCollecting omegaconf>=2.0.5\n",
            "  Downloading https://files.pythonhosted.org/packages/e5/f6/043b6d255dd6fbf2025110cea35b87f4c5100a181681d8eab496269f0d5b/omegaconf-2.0.5-py3-none-any.whl\n",
            "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.6/dist-packages (from hydra-core->fairseq) (3.3.0)\n",
            "Collecting PyYAML>=5.1.*\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/c2/b80047c7ac2478f9501676c988a5411ed5572f35d1beff9cae07d321512c/PyYAML-5.3.1.tar.gz (269kB)\n",
            "\u001b[K     |████████████████████████████████| 276kB 47.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-resources; python_version < \"3.9\"->hydra-core->fairseq) (3.4.0)\n",
            "Building wheels for collected packages: antlr4-python3-runtime, PyYAML\n",
            "  Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.8-cp36-none-any.whl size=141230 sha256=26dba44e111cf2abe4d7b801e5f2472207dda4db4204b7a5c8204f4d6c574873\n",
            "  Stored in directory: /root/.cache/pip/wheels/e3/e2/fa/b78480b448b8579ddf393bebd3f47ee23aa84c89b6a78285c8\n",
            "  Building wheel for PyYAML (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for PyYAML: filename=PyYAML-5.3.1-cp36-cp36m-linux_x86_64.whl size=44619 sha256=9d307af69437b58bfc4459e65deb1f1f20f61ef3d4cbb5710f745aebb5cc5c79\n",
            "  Stored in directory: /root/.cache/pip/wheels/a7/c1/ea/cf5bd31012e735dc1dfea3131a2d5eae7978b251083d6247bd\n",
            "Successfully built antlr4-python3-runtime PyYAML\n",
            "Installing collected packages: portalocker, sacrebleu, antlr4-python3-runtime, PyYAML, omegaconf, hydra-core, fairseq\n",
            "  Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed PyYAML-5.3.1 antlr4-python3-runtime-4.8 fairseq-0.10.1 hydra-core-1.0.4 omegaconf-2.0.5 portalocker-2.0.0 sacrebleu-1.4.14\n",
            "Collecting vncorenlp\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/c2/96a60cf75421ecc740829fa920c617b3dd7fa6791e17554e7c6f3e7d7fca/vncorenlp-1.0.3.tar.gz (2.6MB)\n",
            "\u001b[K     |████████████████████████████████| 2.7MB 14.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from vncorenlp) (2.23.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->vncorenlp) (2020.11.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->vncorenlp) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->vncorenlp) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->vncorenlp) (2.10)\n",
            "Building wheels for collected packages: vncorenlp\n",
            "  Building wheel for vncorenlp (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for vncorenlp: filename=vncorenlp-1.0.3-cp36-none-any.whl size=2645934 sha256=096aefdfe35519716fbf9e9c766f57831dbcfd99b14d22025cbe79b07e2f2c2f\n",
            "  Stored in directory: /root/.cache/pip/wheels/09/54/8b/043667de6091d06a381d7745f44174504a9a4a56ecc9380c54\n",
            "Successfully built vncorenlp\n",
            "Installing collected packages: vncorenlp\n",
            "Successfully installed vncorenlp-1.0.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQsmBDokmc-A"
      },
      "source": [
        "#download PhoBERT base transformer\n",
        "# you must run this cell first time you run this code\n",
        "#!wget https://public.vinai.io/PhoBERT_base_transformers.tar.gz\n",
        "#!tar -xzvf PhoBERT_base_transformers.tar.gz"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncGVmD7VmdCQ"
      },
      "source": [
        "# Download VnCoreNLP-1.1.1.jar & its word segmentation component (i.e. RDRSegmenter) \n",
        "# you must run this cell first time you run this code\n",
        "#!mkdir -p vncorenlp/models/wordsegmenter\n",
        "#!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/VnCoreNLP-1.1.1.jar\n",
        "#!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/vi-vocab\n",
        "#!wget https://raw.githubusercontent.com/vncorenlp/VnCoreNLP/master/models/wordsegmenter/wordsegmenter.rdr\n",
        "#!mv VnCoreNLP-1.1.1.jar vncorenlp/ \n",
        "#!mv vi-vocab vncorenlp/models/wordsegmenter/\n",
        "#!mv wordsegmenter.rdr vncorenlp/models/wordsegmenter/"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "37MAWemD9UGz"
      },
      "source": [
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "import json\n",
        "import numpy as np\n",
        "import pickle\n",
        "import os\n",
        "import torch\n",
        "\n",
        "def convert_lines(df, vocab, bpe, max_sequence_length):\n",
        "    outputs = np.zeros((len(df), max_sequence_length))\n",
        "    \n",
        "    cls_id = 0\n",
        "    eos_id = 2\n",
        "    pad_id = 1\n",
        "\n",
        "    for idx, row in tqdm(df.iterrows(), total=len(df)): \n",
        "        subwords = bpe.encode('<s> '+row.post_message+' </s>')\n",
        "        input_ids = vocab.encode_line(subwords, append_eos=False, add_if_not_exist=False).long().tolist()\n",
        "        if len(input_ids) > max_sequence_length: \n",
        "            input_ids = input_ids[:max_sequence_length] \n",
        "            input_ids[-1] = eos_id\n",
        "        else:\n",
        "            input_ids = input_ids + [pad_id, ]*(max_sequence_length - len(input_ids))\n",
        "        outputs[idx,:] = np.array(input_ids)\n",
        "    return outputs\n",
        "\n",
        "def seed_everything(SEED):\n",
        "    np.random.seed(SEED)\n",
        "    torch.manual_seed(SEED)\n",
        "    torch.cuda.manual_seed(SEED)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1 / (1 + np.exp(-x))"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gKzPea4-HtMr"
      },
      "source": [
        "from vncorenlp import VnCoreNLP\n",
        "rdrsegmenter = VnCoreNLP(path+'/vncorenlp/VnCoreNLP-1.1.1.jar', annotators=\"wseg\", max_heap_size='-Xmx500m') \n",
        "from fairseq.data.encoders.fastbpe import fastBPE\n",
        "from fairseq.data import Dictionary\n",
        "import argparse\n",
        "\n",
        "parser = argparse.ArgumentParser()\n",
        "parser.add_argument('--bpe-codes', \n",
        "    default=path+\"/PhoBERT_base_transformers/bpe.codes\",\n",
        "    required=False,\n",
        "    type=str,\n",
        "    help='path to fastBPE BPE'\n",
        ")\n",
        "args, unknown = parser.parse_known_args()\n",
        "bpe = fastBPE(args)\n",
        "\n",
        "# Load the dictionary\n",
        "vocab = Dictionary()\n",
        "vocab.add_from_file(path+\"/PhoBERT_base_transformers/dict.txt\")"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wJ3s_P6H8vW"
      },
      "source": [
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "tqdm.pandas()\n",
        "from torch import nn\n",
        "import json\n",
        "import numpy as np\n",
        "import pickle\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.model_selection import StratifiedKFold,train_test_split\n",
        "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, roc_auc_score\n",
        "from transformers import *\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "import torch.utils.data\n",
        "import torch.nn.functional as F\n",
        "import argparse\n",
        "from transformers.modeling_utils import * "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fHNjUtxTT46g"
      },
      "source": [
        "#load data\n",
        "import pandas as pd\n",
        "train = pd.read_csv(path+'/VSLP_data/public_train (1).csv')\n",
        "public_test = pd.read_csv(path+'/VSLP_data/public_test.csv')\n",
        "test = pd.read_csv(path+'/VSLP_data/final_private_test_dropped_no_label - final_private_test_dropped_no_label.csv')"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlnYg8p5S9U6"
      },
      "source": [
        "class RobertaForReINTEL(BertPreTrainedModel):\n",
        "   config_class = RobertaConfig\n",
        "   ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = {\n",
        "    'roberta-base': \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-pytorch_model.bin\",\n",
        "    'roberta-large': \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-pytorch_model.bin\",\n",
        "    'roberta-large-mnli': \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-mnli-pytorch_model.bin\",\n",
        "    'distilroberta-base': \"https://s3.amazonaws.com/models.huggingface.co/bert/distilroberta-base-pytorch_model.bin\",\n",
        "    'roberta-base-openai-detector': \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-base-openai-detector-pytorch_model.bin\",\n",
        "    'roberta-large-openai-detector': \"https://s3.amazonaws.com/models.huggingface.co/bert/roberta-large-openai-detector-pytorch_model.bin\",\n",
        "}\n",
        "\n",
        "   pretrained_model_archive_map = ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP\n",
        "   base_model_prefix = \"roberta\"\n",
        "   def __init__(self, config):\n",
        "       super(RobertaForReINTEL, self).__init__(config)\n",
        "       self.num_labels = config.num_labels\n",
        "       self.roberta = RobertaModel(config)\n",
        "       self.qa_outputs = nn.Linear(4*config.hidden_size, self.num_labels)\n",
        "\n",
        "       self.init_weights()\n",
        "\n",
        "   def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None,\n",
        "                start_positions=None, end_positions=None):\n",
        "\n",
        "       outputs = self.roberta(input_ids,\n",
        "                            attention_mask=attention_mask,\n",
        "                            position_ids=position_ids,\n",
        "                            head_mask=head_mask)\n",
        "       cls_output = torch.cat((outputs[2][-1][:,0, ...],outputs[2][-2][:,0, ...], outputs[2][-3][:,0, ...], outputs[2][-4][:,0, ...]),-1)\n",
        "       logits = self.qa_outputs(cls_output)\n",
        "       return logits"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eU548TQzI28X",
        "outputId": "06412352-bc42-4569-945c-fa1f02452317"
      },
      "source": [
        "from transformers import RobertaForSequenceClassification, RobertaConfig, AdamW\n",
        "config = RobertaConfig.from_pretrained(\n",
        "    path+\"/PhoBERT_base_transformers/config.json\",\n",
        "    output_hidden_states=True,\n",
        "    num_labels=1\n",
        ")\n",
        "model_bert = RobertaForReINTEL.from_pretrained(path+'/PhoBERT_base_transformers/model.bin', config=config)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at /content/drive/MyDrive/VLSP_ReINTEL/PhoBERT_base_transformers/model.bin were not used when initializing RobertaForReINTEL: ['lm_head.bias', 'lm_head.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.decoder.bias']\n",
            "- This IS expected if you are initializing RobertaForReINTEL from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForReINTEL from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForReINTEL were not initialized from the model checkpoint at /content/drive/MyDrive/VLSP_ReINTEL/PhoBERT_base_transformers/model.bin and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8XsDE_IOXdIJ",
        "outputId": "078ec176-7362-4733-e6b2-a953548204e3"
      },
      "source": [
        "if torch.cuda.device_count():\n",
        "    print(f\"Training using {torch.cuda.device_count()} gpus\")\n",
        "    model_bert = nn.DataParallel(model_bert)\n",
        "    tsfm = model_bert.module.roberta \n",
        "else:\n",
        "    tsfm = model_bert.roberta"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training using 1 gpus\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "waEjeeIqVh47"
      },
      "source": [
        "kmax_sequence_length= 256\n",
        "kbatch_size= 32\n",
        "kaccumulation_steps= 5\n",
        "kepochs= 5\n",
        "kfold= 0\n",
        "kseed= 42\n",
        "klr= 2e-5"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uv6REkuxQ7bM",
        "outputId": "8dc9132f-b8c1-4bab-baf3-229f5b6e8306"
      },
      "source": [
        "train_df = train[['id','post_message','label']].fillna('none')\n",
        "train_df.post_message = train_df.post_message.progress_apply(lambda x: ' '.join([' '.join(sent) for sent in rdrsegmenter.tokenize(x)])) \n",
        "y = train_df.label.values\n",
        "X_train = convert_lines(train_df, vocab, bpe, kmax_sequence_length)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 4372/4372 [00:31<00:00, 139.32it/s]\n",
            "100%|██████████| 4372/4372 [00:07<00:00, 609.25it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E8T7t-08Tf0Q"
      },
      "source": [
        "param_optimizer = list(model_bert.named_parameters())\n",
        "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
        "optimizer_grouped_parameters = [\n",
        "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
        "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
        "]\n",
        "num_train_optimization_steps = int(kepochs*len(train_df)/kbatch_size/kaccumulation_steps)\n",
        "optimizer = AdamW(optimizer_grouped_parameters, lr=klr, correct_bias=False)  # To reproduce BertAdam specific behavior set correct_bias=False\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=100, num_training_steps=num_train_optimization_steps)  # PyTorch scheduler\n",
        "scheduler0 = get_constant_schedule(optimizer)  # PyTorch scheduler\n"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9TyH57zKWEuD",
        "outputId": "22e9f413-bc95-4be1-9c7c-3e14d954bb84"
      },
      "source": [
        "splits = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=123).split(X_train, y))\n",
        "for fold, (train_idx, val_idx) in enumerate(splits):\n",
        "    if fold != kfold:\n",
        "      continue\n",
        "    train_dataset = torch.utils.data.TensorDataset(torch.tensor(X_train[train_idx],dtype=torch.long), torch.tensor(y[train_idx],dtype=torch.long))\n",
        "    valid_dataset = torch.utils.data.TensorDataset(torch.tensor(X_train[val_idx],dtype=torch.long), torch.tensor(y[val_idx],dtype=torch.long))\n",
        "    tq = tqdm(range(kepochs + 1))\n",
        "    for child in tsfm.children():\n",
        "        for param in child.parameters():\n",
        "            if not param.requires_grad:\n",
        "                print(\"whoopsies\")\n",
        "            param.requires_grad = False\n",
        "    frozen = True\n",
        "    for epoch in tq:\n",
        "\n",
        "        if epoch > 0 and frozen:\n",
        "            for child in tsfm.children():\n",
        "                for param in child.parameters():\n",
        "                    param.requires_grad = True\n",
        "            frozen = False\n",
        "            del scheduler0\n",
        "            torch.cuda.empty_cache()\n",
        "\n",
        "        val_preds = None\n",
        "        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=kbatch_size, shuffle=True)\n",
        "        valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=kbatch_size, shuffle=False)\n",
        "        avg_loss = 0.\n",
        "        avg_accuracy = 0.\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        pbar = tqdm(enumerate(train_loader),total=len(train_loader),leave=False)\n",
        "        for i,(x_batch, y_batch) in pbar:\n",
        "            model_bert.train()\n",
        "            y_pred = model_bert(x_batch.cuda(), attention_mask=(x_batch>0).cuda())\n",
        "            loss =  F.binary_cross_entropy_with_logits(y_pred.view(-1).cuda(),y_batch.float().cuda())\n",
        "            loss = loss.mean()\n",
        "            loss.backward()\n",
        "            if i % kaccumulation_steps == 0 or i == len(pbar) - 1:\n",
        "                optimizer.step()\n",
        "                optimizer.zero_grad()\n",
        "                if not frozen:\n",
        "                    scheduler.step()\n",
        "                else:\n",
        "                    scheduler0.step()\n",
        "            lossf = loss.item()\n",
        "            pbar.set_postfix(loss = lossf)\n",
        "            avg_loss += loss.item() / len(train_loader)\n",
        "\n",
        "        model_bert.eval()\n",
        "        pbar = tqdm(enumerate(valid_loader),total=len(valid_loader),leave=False)\n",
        "        for i,(x_batch, y_batch) in pbar:\n",
        "            y_pred = model_bert(x_batch.cuda(), attention_mask=(x_batch>0).cuda())\n",
        "            y_pred = y_pred.squeeze().detach().cpu().numpy()\n",
        "            val_preds = np.atleast_1d(y_pred) if val_preds is None else np.concatenate([val_preds, np.atleast_1d(y_pred)])\n",
        "        val_preds = sigmoid(val_preds)\n",
        "        score = f1_score(y[val_idx], val_preds > 0.5)\n",
        "        print(f\"\\nAUC = {roc_auc_score(y[val_idx], val_preds):.4f}, F1 score @0.5 = {score:.4f}\")"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  0%|          | 0/6 [00:00<?, ?it/s]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A\n",
            "  0%|          | 0/110 [00:00<?, ?it/s, loss=0.654]\u001b[A\n",
            "  1%|          | 1/110 [00:00<01:11,  1.52it/s, loss=0.654]\u001b[A\n",
            "  1%|          | 1/110 [00:01<01:11,  1.52it/s, loss=0.691]\u001b[A\n",
            "  2%|▏         | 2/110 [00:01<01:04,  1.68it/s, loss=0.691]\u001b[A\n",
            "  2%|▏         | 2/110 [00:01<01:04,  1.68it/s, loss=0.643]\u001b[A\n",
            "  3%|▎         | 3/110 [00:01<00:59,  1.79it/s, loss=0.643]\u001b[A\n",
            "  3%|▎         | 3/110 [00:02<00:59,  1.79it/s, loss=0.653]\u001b[A\n",
            "  4%|▎         | 4/110 [00:02<00:56,  1.89it/s, loss=0.653]\u001b[A\n",
            "  4%|▎         | 4/110 [00:02<00:56,  1.89it/s, loss=0.625]\u001b[A\n",
            "  5%|▍         | 5/110 [00:02<00:53,  1.96it/s, loss=0.625]\u001b[A\n",
            "  5%|▍         | 5/110 [00:02<00:53,  1.96it/s, loss=0.623]\u001b[A\n",
            "  5%|▌         | 6/110 [00:02<00:51,  2.01it/s, loss=0.623]\u001b[A\n",
            "  5%|▌         | 6/110 [00:03<00:51,  2.01it/s, loss=0.651]\u001b[A\n",
            "  6%|▋         | 7/110 [00:03<00:50,  2.05it/s, loss=0.651]\u001b[A\n",
            "  6%|▋         | 7/110 [00:03<00:50,  2.05it/s, loss=0.626]\u001b[A\n",
            "  7%|▋         | 8/110 [00:03<00:49,  2.07it/s, loss=0.626]\u001b[A\n",
            "  7%|▋         | 8/110 [00:04<00:49,  2.07it/s, loss=0.612]\u001b[A\n",
            "  8%|▊         | 9/110 [00:04<00:48,  2.09it/s, loss=0.612]\u001b[A\n",
            "  8%|▊         | 9/110 [00:04<00:48,  2.09it/s, loss=0.582]\u001b[A\n",
            "  9%|▉         | 10/110 [00:04<00:47,  2.11it/s, loss=0.582]\u001b[A\n",
            "  9%|▉         | 10/110 [00:05<00:47,  2.11it/s, loss=0.624]\u001b[A\n",
            " 10%|█         | 11/110 [00:05<00:46,  2.12it/s, loss=0.624]\u001b[A\n",
            " 10%|█         | 11/110 [00:05<00:46,  2.12it/s, loss=0.609]\u001b[A\n",
            " 11%|█         | 12/110 [00:05<00:45,  2.14it/s, loss=0.609]\u001b[A\n",
            " 11%|█         | 12/110 [00:06<00:45,  2.14it/s, loss=0.622]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:06<00:45,  2.14it/s, loss=0.622]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:06<00:45,  2.14it/s, loss=0.58] \u001b[A\n",
            " 13%|█▎        | 14/110 [00:06<00:44,  2.15it/s, loss=0.58]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:07<00:44,  2.15it/s, loss=0.671]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:07<00:44,  2.16it/s, loss=0.671]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:07<00:44,  2.16it/s, loss=0.567]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:07<00:43,  2.15it/s, loss=0.567]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:08<00:43,  2.15it/s, loss=0.609]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:08<00:43,  2.15it/s, loss=0.609]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:08<00:43,  2.15it/s, loss=0.611]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:08<00:42,  2.15it/s, loss=0.611]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:09<00:42,  2.15it/s, loss=0.642]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:09<00:42,  2.15it/s, loss=0.642]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:09<00:42,  2.15it/s, loss=0.577]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:09<00:41,  2.14it/s, loss=0.577]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:09<00:41,  2.14it/s, loss=0.609]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:09<00:41,  2.14it/s, loss=0.609]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:10<00:41,  2.14it/s, loss=0.632]\u001b[A\n",
            " 20%|██        | 22/110 [00:10<00:41,  2.14it/s, loss=0.632]\u001b[A\n",
            " 20%|██        | 22/110 [00:10<00:41,  2.14it/s, loss=0.577]\u001b[A\n",
            " 21%|██        | 23/110 [00:10<00:40,  2.14it/s, loss=0.577]\u001b[A\n",
            " 21%|██        | 23/110 [00:11<00:40,  2.14it/s, loss=0.576]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:11<00:40,  2.14it/s, loss=0.576]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:11<00:40,  2.14it/s, loss=0.602]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:11<00:39,  2.14it/s, loss=0.602]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:12<00:39,  2.14it/s, loss=0.567]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:12<00:39,  2.14it/s, loss=0.567]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:12<00:39,  2.14it/s, loss=0.571]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:12<00:38,  2.14it/s, loss=0.571]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:13<00:38,  2.14it/s, loss=0.555]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:13<00:38,  2.12it/s, loss=0.555]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:13<00:38,  2.12it/s, loss=0.542]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:13<00:38,  2.13it/s, loss=0.542]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:14<00:38,  2.13it/s, loss=0.612]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:14<00:37,  2.12it/s, loss=0.612]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:14<00:37,  2.12it/s, loss=0.555]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:14<00:37,  2.12it/s, loss=0.555]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:15<00:37,  2.12it/s, loss=0.546]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:15<00:36,  2.12it/s, loss=0.546]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:15<00:36,  2.12it/s, loss=0.545]\u001b[A\n",
            " 30%|███       | 33/110 [00:15<00:36,  2.12it/s, loss=0.545]\u001b[A\n",
            " 30%|███       | 33/110 [00:16<00:36,  2.12it/s, loss=0.501]\u001b[A\n",
            " 31%|███       | 34/110 [00:16<00:35,  2.12it/s, loss=0.501]\u001b[A\n",
            " 31%|███       | 34/110 [00:16<00:35,  2.12it/s, loss=0.651]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:16<00:35,  2.12it/s, loss=0.651]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:17<00:35,  2.12it/s, loss=0.543]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:17<00:35,  2.11it/s, loss=0.543]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:17<00:35,  2.11it/s, loss=0.499]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:17<00:34,  2.11it/s, loss=0.499]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:17<00:34,  2.11it/s, loss=0.507]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:17<00:34,  2.11it/s, loss=0.507]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:18<00:34,  2.11it/s, loss=0.516]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:18<00:33,  2.10it/s, loss=0.516]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:18<00:33,  2.10it/s, loss=0.515]\u001b[A\n",
            " 36%|███▋      | 40/110 [00:18<00:33,  2.11it/s, loss=0.515]\u001b[A\n",
            " 36%|███▋      | 40/110 [00:19<00:33,  2.11it/s, loss=0.571]\u001b[A\n",
            " 37%|███▋      | 41/110 [00:19<00:32,  2.09it/s, loss=0.571]\u001b[A\n",
            " 37%|███▋      | 41/110 [00:19<00:32,  2.09it/s, loss=0.498]\u001b[A\n",
            " 38%|███▊      | 42/110 [00:19<00:32,  2.09it/s, loss=0.498]\u001b[A\n",
            " 38%|███▊      | 42/110 [00:20<00:32,  2.09it/s, loss=0.591]\u001b[A\n",
            " 39%|███▉      | 43/110 [00:20<00:32,  2.09it/s, loss=0.591]\u001b[A\n",
            " 39%|███▉      | 43/110 [00:20<00:32,  2.09it/s, loss=0.554]\u001b[A\n",
            " 40%|████      | 44/110 [00:20<00:31,  2.09it/s, loss=0.554]\u001b[A\n",
            " 40%|████      | 44/110 [00:21<00:31,  2.09it/s, loss=0.519]\u001b[A\n",
            " 41%|████      | 45/110 [00:21<00:31,  2.08it/s, loss=0.519]\u001b[A\n",
            " 41%|████      | 45/110 [00:21<00:31,  2.08it/s, loss=0.541]\u001b[A\n",
            " 42%|████▏     | 46/110 [00:21<00:30,  2.08it/s, loss=0.541]\u001b[A\n",
            " 42%|████▏     | 46/110 [00:22<00:30,  2.08it/s, loss=0.513]\u001b[A\n",
            " 43%|████▎     | 47/110 [00:22<00:30,  2.08it/s, loss=0.513]\u001b[A\n",
            " 43%|████▎     | 47/110 [00:22<00:30,  2.08it/s, loss=0.412]\u001b[A\n",
            " 44%|████▎     | 48/110 [00:22<00:29,  2.07it/s, loss=0.412]\u001b[A\n",
            " 44%|████▎     | 48/110 [00:23<00:29,  2.07it/s, loss=0.495]\u001b[A\n",
            " 45%|████▍     | 49/110 [00:23<00:29,  2.07it/s, loss=0.495]\u001b[A\n",
            " 45%|████▍     | 49/110 [00:23<00:29,  2.07it/s, loss=0.501]\u001b[A\n",
            " 45%|████▌     | 50/110 [00:23<00:28,  2.07it/s, loss=0.501]\u001b[A\n",
            " 45%|████▌     | 50/110 [00:24<00:28,  2.07it/s, loss=0.502]\u001b[A\n",
            " 46%|████▋     | 51/110 [00:24<00:28,  2.07it/s, loss=0.502]\u001b[A\n",
            " 46%|████▋     | 51/110 [00:24<00:28,  2.07it/s, loss=0.499]\u001b[A\n",
            " 47%|████▋     | 52/110 [00:24<00:28,  2.06it/s, loss=0.499]\u001b[A\n",
            " 47%|████▋     | 52/110 [00:25<00:28,  2.06it/s, loss=0.54] \u001b[A\n",
            " 48%|████▊     | 53/110 [00:25<00:27,  2.06it/s, loss=0.54]\u001b[A\n",
            " 48%|████▊     | 53/110 [00:25<00:27,  2.06it/s, loss=0.578]\u001b[A\n",
            " 49%|████▉     | 54/110 [00:25<00:27,  2.05it/s, loss=0.578]\u001b[A\n",
            " 49%|████▉     | 54/110 [00:26<00:27,  2.05it/s, loss=0.44] \u001b[A\n",
            " 50%|█████     | 55/110 [00:26<00:26,  2.05it/s, loss=0.44]\u001b[A\n",
            " 50%|█████     | 55/110 [00:26<00:26,  2.05it/s, loss=0.53]\u001b[A\n",
            " 51%|█████     | 56/110 [00:26<00:26,  2.05it/s, loss=0.53]\u001b[A\n",
            " 51%|█████     | 56/110 [00:27<00:26,  2.05it/s, loss=0.447]\u001b[A\n",
            " 52%|█████▏    | 57/110 [00:27<00:25,  2.05it/s, loss=0.447]\u001b[A\n",
            " 52%|█████▏    | 57/110 [00:27<00:25,  2.05it/s, loss=0.582]\u001b[A\n",
            " 53%|█████▎    | 58/110 [00:27<00:25,  2.04it/s, loss=0.582]\u001b[A\n",
            " 53%|█████▎    | 58/110 [00:28<00:25,  2.04it/s, loss=0.577]\u001b[A\n",
            " 54%|█████▎    | 59/110 [00:28<00:24,  2.05it/s, loss=0.577]\u001b[A\n",
            " 54%|█████▎    | 59/110 [00:28<00:24,  2.05it/s, loss=0.569]\u001b[A\n",
            " 55%|█████▍    | 60/110 [00:28<00:24,  2.06it/s, loss=0.569]\u001b[A\n",
            " 55%|█████▍    | 60/110 [00:29<00:24,  2.06it/s, loss=0.567]\u001b[A\n",
            " 55%|█████▌    | 61/110 [00:29<00:23,  2.06it/s, loss=0.567]\u001b[A\n",
            " 55%|█████▌    | 61/110 [00:29<00:23,  2.06it/s, loss=0.533]\u001b[A\n",
            " 56%|█████▋    | 62/110 [00:29<00:23,  2.05it/s, loss=0.533]\u001b[A\n",
            " 56%|█████▋    | 62/110 [00:30<00:23,  2.05it/s, loss=0.477]\u001b[A\n",
            " 57%|█████▋    | 63/110 [00:30<00:22,  2.05it/s, loss=0.477]\u001b[A\n",
            " 57%|█████▋    | 63/110 [00:30<00:22,  2.05it/s, loss=0.693]\u001b[A\n",
            " 58%|█████▊    | 64/110 [00:30<00:22,  2.05it/s, loss=0.693]\u001b[A\n",
            " 58%|█████▊    | 64/110 [00:31<00:22,  2.05it/s, loss=0.528]\u001b[A\n",
            " 59%|█████▉    | 65/110 [00:31<00:21,  2.05it/s, loss=0.528]\u001b[A\n",
            " 59%|█████▉    | 65/110 [00:31<00:21,  2.05it/s, loss=0.416]\u001b[A\n",
            " 60%|██████    | 66/110 [00:31<00:21,  2.05it/s, loss=0.416]\u001b[A\n",
            " 60%|██████    | 66/110 [00:32<00:21,  2.05it/s, loss=0.554]\u001b[A\n",
            " 61%|██████    | 67/110 [00:32<00:21,  2.05it/s, loss=0.554]\u001b[A\n",
            " 61%|██████    | 67/110 [00:32<00:21,  2.05it/s, loss=0.469]\u001b[A\n",
            " 62%|██████▏   | 68/110 [00:32<00:20,  2.04it/s, loss=0.469]\u001b[A\n",
            " 62%|██████▏   | 68/110 [00:33<00:20,  2.04it/s, loss=0.558]\u001b[A\n",
            " 63%|██████▎   | 69/110 [00:33<00:20,  2.04it/s, loss=0.558]\u001b[A\n",
            " 63%|██████▎   | 69/110 [00:33<00:20,  2.04it/s, loss=0.508]\u001b[A\n",
            " 64%|██████▎   | 70/110 [00:33<00:19,  2.04it/s, loss=0.508]\u001b[A\n",
            " 64%|██████▎   | 70/110 [00:34<00:19,  2.04it/s, loss=0.518]\u001b[A\n",
            " 65%|██████▍   | 71/110 [00:34<00:19,  2.03it/s, loss=0.518]\u001b[A\n",
            " 65%|██████▍   | 71/110 [00:34<00:19,  2.03it/s, loss=0.572]\u001b[A\n",
            " 65%|██████▌   | 72/110 [00:34<00:18,  2.04it/s, loss=0.572]\u001b[A\n",
            " 65%|██████▌   | 72/110 [00:34<00:18,  2.04it/s, loss=0.428]\u001b[A\n",
            " 66%|██████▋   | 73/110 [00:35<00:18,  2.03it/s, loss=0.428]\u001b[A\n",
            " 66%|██████▋   | 73/110 [00:35<00:18,  2.03it/s, loss=0.406]\u001b[A\n",
            " 67%|██████▋   | 74/110 [00:35<00:17,  2.03it/s, loss=0.406]\u001b[A\n",
            " 67%|██████▋   | 74/110 [00:35<00:17,  2.03it/s, loss=0.549]\u001b[A\n",
            " 68%|██████▊   | 75/110 [00:35<00:17,  2.03it/s, loss=0.549]\u001b[A\n",
            " 68%|██████▊   | 75/110 [00:36<00:17,  2.03it/s, loss=0.389]\u001b[A\n",
            " 69%|██████▉   | 76/110 [00:36<00:16,  2.02it/s, loss=0.389]\u001b[A\n",
            " 69%|██████▉   | 76/110 [00:36<00:16,  2.02it/s, loss=0.289]\u001b[A\n",
            " 70%|███████   | 77/110 [00:36<00:16,  2.03it/s, loss=0.289]\u001b[A\n",
            " 70%|███████   | 77/110 [00:37<00:16,  2.03it/s, loss=0.462]\u001b[A\n",
            " 71%|███████   | 78/110 [00:37<00:15,  2.02it/s, loss=0.462]\u001b[A\n",
            " 71%|███████   | 78/110 [00:37<00:15,  2.02it/s, loss=0.503]\u001b[A\n",
            " 72%|███████▏  | 79/110 [00:37<00:15,  2.02it/s, loss=0.503]\u001b[A\n",
            " 72%|███████▏  | 79/110 [00:38<00:15,  2.02it/s, loss=0.475]\u001b[A\n",
            " 73%|███████▎  | 80/110 [00:38<00:14,  2.02it/s, loss=0.475]\u001b[A\n",
            " 73%|███████▎  | 80/110 [00:38<00:14,  2.02it/s, loss=0.518]\u001b[A\n",
            " 74%|███████▎  | 81/110 [00:38<00:14,  2.02it/s, loss=0.518]\u001b[A\n",
            " 74%|███████▎  | 81/110 [00:39<00:14,  2.02it/s, loss=0.52] \u001b[A\n",
            " 75%|███████▍  | 82/110 [00:39<00:13,  2.01it/s, loss=0.52]\u001b[A\n",
            " 75%|███████▍  | 82/110 [00:39<00:13,  2.01it/s, loss=0.515]\u001b[A\n",
            " 75%|███████▌  | 83/110 [00:39<00:13,  2.02it/s, loss=0.515]\u001b[A\n",
            " 75%|███████▌  | 83/110 [00:40<00:13,  2.02it/s, loss=0.381]\u001b[A\n",
            " 76%|███████▋  | 84/110 [00:40<00:12,  2.01it/s, loss=0.381]\u001b[A\n",
            " 76%|███████▋  | 84/110 [00:40<00:12,  2.01it/s, loss=0.355]\u001b[A\n",
            " 77%|███████▋  | 85/110 [00:40<00:12,  2.00it/s, loss=0.355]\u001b[A\n",
            " 77%|███████▋  | 85/110 [00:41<00:12,  2.00it/s, loss=0.468]\u001b[A\n",
            " 78%|███████▊  | 86/110 [00:41<00:12,  1.99it/s, loss=0.468]\u001b[A\n",
            " 78%|███████▊  | 86/110 [00:41<00:12,  1.99it/s, loss=0.472]\u001b[A\n",
            " 79%|███████▉  | 87/110 [00:41<00:11,  2.00it/s, loss=0.472]\u001b[A\n",
            " 79%|███████▉  | 87/110 [00:42<00:11,  2.00it/s, loss=0.402]\u001b[A\n",
            " 80%|████████  | 88/110 [00:42<00:10,  2.00it/s, loss=0.402]\u001b[A\n",
            " 80%|████████  | 88/110 [00:42<00:10,  2.00it/s, loss=0.494]\u001b[A\n",
            " 81%|████████  | 89/110 [00:42<00:10,  2.01it/s, loss=0.494]\u001b[A\n",
            " 81%|████████  | 89/110 [00:43<00:10,  2.01it/s, loss=0.369]\u001b[A\n",
            " 82%|████████▏ | 90/110 [00:43<00:09,  2.01it/s, loss=0.369]\u001b[A\n",
            " 82%|████████▏ | 90/110 [00:43<00:09,  2.01it/s, loss=0.505]\u001b[A\n",
            " 83%|████████▎ | 91/110 [00:43<00:09,  2.01it/s, loss=0.505]\u001b[A\n",
            " 83%|████████▎ | 91/110 [00:44<00:09,  2.01it/s, loss=0.446]\u001b[A\n",
            " 84%|████████▎ | 92/110 [00:44<00:08,  2.01it/s, loss=0.446]\u001b[A\n",
            " 84%|████████▎ | 92/110 [00:44<00:08,  2.01it/s, loss=0.379]\u001b[A\n",
            " 85%|████████▍ | 93/110 [00:44<00:08,  2.02it/s, loss=0.379]\u001b[A\n",
            " 85%|████████▍ | 93/110 [00:45<00:08,  2.02it/s, loss=0.401]\u001b[A\n",
            " 85%|████████▌ | 94/110 [00:45<00:07,  2.01it/s, loss=0.401]\u001b[A\n",
            " 85%|████████▌ | 94/110 [00:45<00:07,  2.01it/s, loss=0.513]\u001b[A\n",
            " 86%|████████▋ | 95/110 [00:45<00:07,  2.02it/s, loss=0.513]\u001b[A\n",
            " 86%|████████▋ | 95/110 [00:46<00:07,  2.02it/s, loss=0.385]\u001b[A\n",
            " 87%|████████▋ | 96/110 [00:46<00:06,  2.01it/s, loss=0.385]\u001b[A\n",
            " 87%|████████▋ | 96/110 [00:46<00:06,  2.01it/s, loss=0.456]\u001b[A\n",
            " 88%|████████▊ | 97/110 [00:46<00:06,  2.02it/s, loss=0.456]\u001b[A\n",
            " 88%|████████▊ | 97/110 [00:47<00:06,  2.02it/s, loss=0.403]\u001b[A\n",
            " 89%|████████▉ | 98/110 [00:47<00:05,  2.01it/s, loss=0.403]\u001b[A\n",
            " 89%|████████▉ | 98/110 [00:47<00:05,  2.01it/s, loss=0.501]\u001b[A\n",
            " 90%|█████████ | 99/110 [00:47<00:05,  2.00it/s, loss=0.501]\u001b[A\n",
            " 90%|█████████ | 99/110 [00:48<00:05,  2.00it/s, loss=0.385]\u001b[A\n",
            " 91%|█████████ | 100/110 [00:48<00:04,  2.00it/s, loss=0.385]\u001b[A\n",
            " 91%|█████████ | 100/110 [00:48<00:04,  2.00it/s, loss=0.456]\u001b[A\n",
            " 92%|█████████▏| 101/110 [00:48<00:04,  2.00it/s, loss=0.456]\u001b[A\n",
            " 92%|█████████▏| 101/110 [00:49<00:04,  2.00it/s, loss=0.54] \u001b[A\n",
            " 93%|█████████▎| 102/110 [00:49<00:04,  1.99it/s, loss=0.54]\u001b[A\n",
            " 93%|█████████▎| 102/110 [00:49<00:04,  1.99it/s, loss=0.546]\u001b[A\n",
            " 94%|█████████▎| 103/110 [00:49<00:03,  1.99it/s, loss=0.546]\u001b[A\n",
            " 94%|█████████▎| 103/110 [00:50<00:03,  1.99it/s, loss=0.547]\u001b[A\n",
            " 95%|█████████▍| 104/110 [00:50<00:03,  1.99it/s, loss=0.547]\u001b[A\n",
            " 95%|█████████▍| 104/110 [00:50<00:03,  1.99it/s, loss=0.534]\u001b[A\n",
            " 95%|█████████▌| 105/110 [00:50<00:02,  1.98it/s, loss=0.534]\u001b[A\n",
            " 95%|█████████▌| 105/110 [00:51<00:02,  1.98it/s, loss=0.562]\u001b[A\n",
            " 96%|█████████▋| 106/110 [00:51<00:02,  1.97it/s, loss=0.562]\u001b[A\n",
            " 96%|█████████▋| 106/110 [00:51<00:02,  1.97it/s, loss=0.426]\u001b[A\n",
            " 97%|█████████▋| 107/110 [00:51<00:01,  1.98it/s, loss=0.426]\u001b[A\n",
            " 97%|█████████▋| 107/110 [00:52<00:01,  1.98it/s, loss=0.496]\u001b[A\n",
            " 98%|█████████▊| 108/110 [00:52<00:01,  1.97it/s, loss=0.496]\u001b[A\n",
            " 98%|█████████▊| 108/110 [00:52<00:01,  1.97it/s, loss=0.462]\u001b[A\n",
            " 99%|█████████▉| 109/110 [00:52<00:00,  1.97it/s, loss=0.462]\u001b[A\n",
            " 99%|█████████▉| 109/110 [00:53<00:00,  1.97it/s, loss=0.214]\u001b[A\n",
            "100%|██████████| 110/110 [00:53<00:00,  2.50it/s, loss=0.214]\u001b[A\n",
            "                                                             \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:12,  2.10it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:00<00:12,  2.09it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:12,  2.08it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:01<00:11,  2.06it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:11,  2.05it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:02<00:10,  2.05it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:10,  2.03it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:03<00:09,  2.03it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:09,  2.03it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:04<00:08,  2.02it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  2.03it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:05<00:07,  2.02it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  2.02it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:06<00:06,  2.01it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  2.02it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:07<00:05,  2.01it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  2.00it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:08<00:04,  2.01it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:09<00:04,  2.00it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:09<00:04,  2.00it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:10<00:03,  2.01it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:10<00:03,  2.00it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:11<00:02,  1.99it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:11<00:02,  2.00it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:12<00:01,  2.00it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:12<00:01,  1.99it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:13<00:00,  1.99it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:13<00:00,  2.47it/s]\u001b[A\n",
            " 17%|█▋        | 1/6 [01:06<05:33, 66.75s/it]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.5842, F1 score @0.5 = 0.0000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/110 [00:01<?, ?it/s, loss=0.537]\u001b[A\n",
            "  1%|          | 1/110 [00:01<02:44,  1.51s/it, loss=0.537]\u001b[A\n",
            "  1%|          | 1/110 [00:03<02:44,  1.51s/it, loss=0.46] \u001b[A\n",
            "  2%|▏         | 2/110 [00:03<02:42,  1.51s/it, loss=0.46]\u001b[A\n",
            "  2%|▏         | 2/110 [00:04<02:42,  1.51s/it, loss=0.481]\u001b[A\n",
            "  3%|▎         | 3/110 [00:04<02:41,  1.51s/it, loss=0.481]\u001b[A\n",
            "  3%|▎         | 3/110 [00:06<02:41,  1.51s/it, loss=0.534]\u001b[A\n",
            "  4%|▎         | 4/110 [00:06<02:39,  1.51s/it, loss=0.534]\u001b[A\n",
            "  4%|▎         | 4/110 [00:07<02:39,  1.51s/it, loss=0.499]\u001b[A\n",
            "  5%|▍         | 5/110 [00:07<02:38,  1.51s/it, loss=0.499]\u001b[A\n",
            "  5%|▍         | 5/110 [00:09<02:38,  1.51s/it, loss=0.502]\u001b[A\n",
            "  5%|▌         | 6/110 [00:09<02:39,  1.53s/it, loss=0.502]\u001b[A\n",
            "  5%|▌         | 6/110 [00:10<02:39,  1.53s/it, loss=0.562]\u001b[A\n",
            "  6%|▋         | 7/110 [00:10<02:37,  1.53s/it, loss=0.562]\u001b[A\n",
            "  6%|▋         | 7/110 [00:12<02:37,  1.53s/it, loss=0.483]\u001b[A\n",
            "  7%|▋         | 8/110 [00:12<02:36,  1.53s/it, loss=0.483]\u001b[A\n",
            "  7%|▋         | 8/110 [00:13<02:36,  1.53s/it, loss=0.533]\u001b[A\n",
            "  8%|▊         | 9/110 [00:13<02:35,  1.54s/it, loss=0.533]\u001b[A\n",
            "  8%|▊         | 9/110 [00:15<02:35,  1.54s/it, loss=0.535]\u001b[A\n",
            "  9%|▉         | 10/110 [00:15<02:34,  1.54s/it, loss=0.535]\u001b[A\n",
            "  9%|▉         | 10/110 [00:16<02:34,  1.54s/it, loss=0.467]\u001b[A\n",
            " 10%|█         | 11/110 [00:16<02:34,  1.56s/it, loss=0.467]\u001b[A\n",
            " 10%|█         | 11/110 [00:18<02:34,  1.56s/it, loss=0.486]\u001b[A\n",
            " 11%|█         | 12/110 [00:18<02:33,  1.56s/it, loss=0.486]\u001b[A\n",
            " 11%|█         | 12/110 [00:20<02:33,  1.56s/it, loss=0.49] \u001b[A\n",
            " 12%|█▏        | 13/110 [00:20<02:31,  1.57s/it, loss=0.49]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:21<02:31,  1.57s/it, loss=0.591]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:21<02:30,  1.57s/it, loss=0.591]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:23<02:30,  1.57s/it, loss=0.468]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:23<02:29,  1.57s/it, loss=0.468]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:24<02:29,  1.57s/it, loss=0.591]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:24<02:29,  1.59s/it, loss=0.591]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:26<02:29,  1.59s/it, loss=0.487]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:26<02:26,  1.58s/it, loss=0.487]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:27<02:26,  1.58s/it, loss=0.322]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:27<02:24,  1.57s/it, loss=0.322]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:29<02:24,  1.57s/it, loss=0.483]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:29<02:22,  1.57s/it, loss=0.483]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:31<02:22,  1.57s/it, loss=0.503]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:31<02:20,  1.56s/it, loss=0.503]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:32<02:20,  1.56s/it, loss=0.397]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:32<02:19,  1.57s/it, loss=0.397]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:34<02:19,  1.57s/it, loss=0.526]\u001b[A\n",
            " 20%|██        | 22/110 [00:34<02:17,  1.56s/it, loss=0.526]\u001b[A\n",
            " 20%|██        | 22/110 [00:35<02:17,  1.56s/it, loss=0.534]\u001b[A\n",
            " 21%|██        | 23/110 [00:35<02:14,  1.55s/it, loss=0.534]\u001b[A\n",
            " 21%|██        | 23/110 [00:37<02:14,  1.55s/it, loss=0.569]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:37<02:12,  1.54s/it, loss=0.569]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:38<02:12,  1.54s/it, loss=0.517]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:38<02:10,  1.54s/it, loss=0.517]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:40<02:10,  1.54s/it, loss=0.402]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:40<02:09,  1.55s/it, loss=0.402]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:41<02:09,  1.55s/it, loss=0.433]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:41<02:07,  1.53s/it, loss=0.433]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:43<02:07,  1.53s/it, loss=0.361]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:43<02:05,  1.53s/it, loss=0.361]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:44<02:05,  1.53s/it, loss=0.434]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:44<02:03,  1.52s/it, loss=0.434]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:46<02:03,  1.52s/it, loss=0.455]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:46<02:01,  1.52s/it, loss=0.455]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:47<02:01,  1.52s/it, loss=0.365]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:47<02:00,  1.53s/it, loss=0.365]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:49<02:00,  1.53s/it, loss=0.698]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:49<01:58,  1.52s/it, loss=0.698]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:50<01:58,  1.52s/it, loss=0.191]\u001b[A\n",
            " 30%|███       | 33/110 [00:50<01:56,  1.51s/it, loss=0.191]\u001b[A\n",
            " 30%|███       | 33/110 [00:52<01:56,  1.51s/it, loss=0.27] \u001b[A\n",
            " 31%|███       | 34/110 [00:52<01:54,  1.51s/it, loss=0.27]\u001b[A\n",
            " 31%|███       | 34/110 [00:53<01:54,  1.51s/it, loss=0.627]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:53<01:52,  1.51s/it, loss=0.627]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:55<01:52,  1.51s/it, loss=0.456]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:55<01:52,  1.52s/it, loss=0.456]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:56<01:52,  1.52s/it, loss=0.187]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:56<01:50,  1.51s/it, loss=0.187]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:58<01:50,  1.51s/it, loss=0.263]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:58<01:48,  1.51s/it, loss=0.263]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:59<01:48,  1.51s/it, loss=0.287]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:59<01:46,  1.50s/it, loss=0.287]\u001b[A\n",
            " 35%|███▌      | 39/110 [01:01<01:46,  1.50s/it, loss=0.637]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:01<01:45,  1.50s/it, loss=0.637]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:02<01:45,  1.50s/it, loss=0.474]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:02<01:44,  1.52s/it, loss=0.474]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:04<01:44,  1.52s/it, loss=0.47] \u001b[A\n",
            " 38%|███▊      | 42/110 [01:04<01:42,  1.51s/it, loss=0.47]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:05<01:42,  1.51s/it, loss=0.644]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:05<01:41,  1.51s/it, loss=0.644]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:07<01:41,  1.51s/it, loss=0.486]\u001b[A\n",
            " 40%|████      | 44/110 [01:07<01:39,  1.50s/it, loss=0.486]\u001b[A\n",
            " 40%|████      | 44/110 [01:08<01:39,  1.50s/it, loss=0.517]\u001b[A\n",
            " 41%|████      | 45/110 [01:08<01:37,  1.50s/it, loss=0.517]\u001b[A\n",
            " 41%|████      | 45/110 [01:10<01:37,  1.50s/it, loss=0.879]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:10<01:37,  1.52s/it, loss=0.879]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:12<01:37,  1.52s/it, loss=0.59] \u001b[A\n",
            " 43%|████▎     | 47/110 [01:12<01:35,  1.51s/it, loss=0.59]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:13<01:35,  1.51s/it, loss=0.0977]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:13<01:33,  1.51s/it, loss=0.0977]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:15<01:33,  1.51s/it, loss=0.355] \u001b[A\n",
            " 45%|████▍     | 49/110 [01:15<01:32,  1.51s/it, loss=0.355]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:16<01:32,  1.51s/it, loss=0.542]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:16<01:30,  1.51s/it, loss=0.542]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:18<01:30,  1.51s/it, loss=0.286]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:18<01:29,  1.52s/it, loss=0.286]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:19<01:29,  1.52s/it, loss=0.696]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:19<01:27,  1.51s/it, loss=0.696]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:21<01:27,  1.51s/it, loss=0.604]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:21<01:26,  1.51s/it, loss=0.604]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:22<01:26,  1.51s/it, loss=0.404]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:22<01:24,  1.51s/it, loss=0.404]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:24<01:24,  1.51s/it, loss=0.621]\u001b[A\n",
            " 50%|█████     | 55/110 [01:24<01:23,  1.51s/it, loss=0.621]\u001b[A\n",
            " 50%|█████     | 55/110 [01:25<01:23,  1.51s/it, loss=0.728]\u001b[A\n",
            " 51%|█████     | 56/110 [01:25<01:22,  1.53s/it, loss=0.728]\u001b[A\n",
            " 51%|█████     | 56/110 [01:27<01:22,  1.53s/it, loss=0.253]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:27<01:20,  1.52s/it, loss=0.253]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:28<01:20,  1.52s/it, loss=0.674]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:28<01:18,  1.52s/it, loss=0.674]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:30<01:18,  1.52s/it, loss=0.327]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:30<01:17,  1.52s/it, loss=0.327]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:31<01:17,  1.52s/it, loss=0.583]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:31<01:15,  1.52s/it, loss=0.583]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:33<01:15,  1.52s/it, loss=0.488]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:33<01:14,  1.53s/it, loss=0.488]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:34<01:14,  1.53s/it, loss=0.415]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:34<01:13,  1.53s/it, loss=0.415]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:36<01:13,  1.53s/it, loss=0.469]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:36<01:11,  1.53s/it, loss=0.469]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:37<01:11,  1.53s/it, loss=0.749]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:37<01:10,  1.53s/it, loss=0.749]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:39<01:10,  1.53s/it, loss=0.441]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:39<01:08,  1.53s/it, loss=0.441]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:40<01:08,  1.53s/it, loss=0.41] \u001b[A\n",
            " 60%|██████    | 66/110 [01:40<01:07,  1.54s/it, loss=0.41]\u001b[A\n",
            " 60%|██████    | 66/110 [01:42<01:07,  1.54s/it, loss=0.397]\u001b[A\n",
            " 61%|██████    | 67/110 [01:42<01:06,  1.54s/it, loss=0.397]\u001b[A\n",
            " 61%|██████    | 67/110 [01:44<01:06,  1.54s/it, loss=0.59] \u001b[A\n",
            " 62%|██████▏   | 68/110 [01:44<01:04,  1.53s/it, loss=0.59]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:45<01:04,  1.53s/it, loss=0.577]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:45<01:02,  1.53s/it, loss=0.577]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:47<01:02,  1.53s/it, loss=0.281]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:47<01:01,  1.53s/it, loss=0.281]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:48<01:01,  1.53s/it, loss=0.829]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:48<01:00,  1.54s/it, loss=0.829]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:50<01:00,  1.54s/it, loss=0.596]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:50<00:58,  1.54s/it, loss=0.596]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:51<00:58,  1.54s/it, loss=0.56] \u001b[A\n",
            " 66%|██████▋   | 73/110 [01:51<00:56,  1.54s/it, loss=0.56]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:53<00:56,  1.54s/it, loss=0.317]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:53<00:55,  1.53s/it, loss=0.317]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:54<00:55,  1.53s/it, loss=0.494]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:54<00:53,  1.53s/it, loss=0.494]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:56<00:53,  1.53s/it, loss=0.301]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:56<00:52,  1.54s/it, loss=0.301]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:57<00:52,  1.54s/it, loss=0.373]\u001b[A\n",
            " 70%|███████   | 77/110 [01:57<00:50,  1.53s/it, loss=0.373]\u001b[A\n",
            " 70%|███████   | 77/110 [01:59<00:50,  1.53s/it, loss=0.345]\u001b[A\n",
            " 71%|███████   | 78/110 [01:59<00:49,  1.53s/it, loss=0.345]\u001b[A\n",
            " 71%|███████   | 78/110 [02:00<00:49,  1.53s/it, loss=0.508]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:00<00:47,  1.53s/it, loss=0.508]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:02<00:47,  1.53s/it, loss=0.414]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:02<00:45,  1.53s/it, loss=0.414]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:03<00:45,  1.53s/it, loss=0.57] \u001b[A\n",
            " 74%|███████▎  | 81/110 [02:03<00:44,  1.54s/it, loss=0.57]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:05<00:44,  1.54s/it, loss=0.439]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:05<00:42,  1.53s/it, loss=0.439]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:07<00:42,  1.53s/it, loss=0.398]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:07<00:41,  1.53s/it, loss=0.398]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:08<00:41,  1.53s/it, loss=0.431]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:08<00:39,  1.52s/it, loss=0.431]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:10<00:39,  1.52s/it, loss=0.426]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:10<00:37,  1.52s/it, loss=0.426]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:11<00:37,  1.52s/it, loss=0.473]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:11<00:36,  1.53s/it, loss=0.473]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:13<00:36,  1.53s/it, loss=0.391]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:13<00:35,  1.53s/it, loss=0.391]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:14<00:35,  1.53s/it, loss=0.473]\u001b[A\n",
            " 80%|████████  | 88/110 [02:14<00:33,  1.52s/it, loss=0.473]\u001b[A\n",
            " 80%|████████  | 88/110 [02:16<00:33,  1.52s/it, loss=0.321]\u001b[A\n",
            " 81%|████████  | 89/110 [02:16<00:31,  1.52s/it, loss=0.321]\u001b[A\n",
            " 81%|████████  | 89/110 [02:17<00:31,  1.52s/it, loss=0.489]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:17<00:30,  1.52s/it, loss=0.489]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:19<00:30,  1.52s/it, loss=0.441]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:19<00:29,  1.53s/it, loss=0.441]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:20<00:29,  1.53s/it, loss=0.36] \u001b[A\n",
            " 84%|████████▎ | 92/110 [02:20<00:27,  1.52s/it, loss=0.36]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:22<00:27,  1.52s/it, loss=0.553]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:22<00:25,  1.52s/it, loss=0.553]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:23<00:25,  1.52s/it, loss=0.284]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:23<00:24,  1.51s/it, loss=0.284]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:25<00:24,  1.51s/it, loss=0.566]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:25<00:22,  1.51s/it, loss=0.566]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:26<00:22,  1.51s/it, loss=0.611]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:26<00:21,  1.53s/it, loss=0.611]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:28<00:21,  1.53s/it, loss=0.361]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:28<00:19,  1.52s/it, loss=0.361]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:29<00:19,  1.52s/it, loss=0.486]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:29<00:18,  1.52s/it, loss=0.486]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:31<00:18,  1.52s/it, loss=0.437]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:31<00:16,  1.51s/it, loss=0.437]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:32<00:16,  1.51s/it, loss=0.314]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:32<00:15,  1.51s/it, loss=0.314]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:34<00:15,  1.51s/it, loss=0.465]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:34<00:13,  1.53s/it, loss=0.465]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:35<00:13,  1.53s/it, loss=0.316]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:35<00:12,  1.52s/it, loss=0.316]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:37<00:12,  1.52s/it, loss=0.7]  \u001b[A\n",
            " 94%|█████████▎| 103/110 [02:37<00:10,  1.52s/it, loss=0.7]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:38<00:10,  1.52s/it, loss=0.366]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:38<00:09,  1.52s/it, loss=0.366]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:40<00:09,  1.52s/it, loss=0.357]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:40<00:07,  1.51s/it, loss=0.357]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:41<00:07,  1.51s/it, loss=0.512]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:41<00:06,  1.53s/it, loss=0.512]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:43<00:06,  1.53s/it, loss=0.577]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:43<00:04,  1.52s/it, loss=0.577]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:45<00:04,  1.52s/it, loss=0.466]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:45<00:03,  1.52s/it, loss=0.466]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:46<00:03,  1.52s/it, loss=0.392]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.51s/it, loss=0.392]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.51s/it, loss=0.295]\u001b[A\n",
            "100%|██████████| 110/110 [02:47<00:00,  1.21s/it, loss=0.295]\u001b[A\n",
            "                                                             \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:13,  1.93it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:13,  1.91it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:12,  1.90it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:03<00:11,  1.90it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:11,  1.90it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:04<00:10,  1.89it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:10,  1.90it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:05<00:09,  1.90it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  1.90it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:06<00:08,  1.90it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  1.90it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:07<00:07,  1.90it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  1.90it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:08<00:06,  1.90it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  1.90it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:09<00:05,  1.90it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:10<00:04,  1.90it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:10<00:04,  1.90it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:11<00:03,  1.90it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:11<00:03,  1.90it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:12<00:02,  1.89it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:12<00:02,  1.90it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:13<00:01,  1.89it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:14<00:00,  1.90it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:14<00:00,  2.35it/s]\u001b[A\n",
            " 33%|███▎      | 2/6 [04:08<06:44, 101.17s/it]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.6499, F1 score @0.5 = 0.0000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/110 [00:01<?, ?it/s, loss=0.272]\u001b[A\n",
            "  1%|          | 1/110 [00:01<02:48,  1.55s/it, loss=0.272]\u001b[A\n",
            "  1%|          | 1/110 [00:03<02:48,  1.55s/it, loss=0.364]\u001b[A\n",
            "  2%|▏         | 2/110 [00:03<02:45,  1.54s/it, loss=0.364]\u001b[A\n",
            "  2%|▏         | 2/110 [00:04<02:45,  1.54s/it, loss=0.557]\u001b[A\n",
            "  3%|▎         | 3/110 [00:04<02:43,  1.53s/it, loss=0.557]\u001b[A\n",
            "  3%|▎         | 3/110 [00:06<02:43,  1.53s/it, loss=0.204]\u001b[A\n",
            "  4%|▎         | 4/110 [00:06<02:41,  1.52s/it, loss=0.204]\u001b[A\n",
            "  4%|▎         | 4/110 [00:07<02:41,  1.52s/it, loss=0.439]\u001b[A\n",
            "  5%|▍         | 5/110 [00:07<02:39,  1.52s/it, loss=0.439]\u001b[A\n",
            "  5%|▍         | 5/110 [00:09<02:39,  1.52s/it, loss=0.624]\u001b[A\n",
            "  5%|▌         | 6/110 [00:09<02:39,  1.53s/it, loss=0.624]\u001b[A\n",
            "  5%|▌         | 6/110 [00:10<02:39,  1.53s/it, loss=0.456]\u001b[A\n",
            "  6%|▋         | 7/110 [00:10<02:37,  1.52s/it, loss=0.456]\u001b[A\n",
            "  6%|▋         | 7/110 [00:12<02:37,  1.52s/it, loss=0.4]  \u001b[A\n",
            "  7%|▋         | 8/110 [00:12<02:35,  1.52s/it, loss=0.4]\u001b[A\n",
            "  7%|▋         | 8/110 [00:13<02:35,  1.52s/it, loss=0.367]\u001b[A\n",
            "  8%|▊         | 9/110 [00:13<02:33,  1.52s/it, loss=0.367]\u001b[A\n",
            "  8%|▊         | 9/110 [00:15<02:33,  1.52s/it, loss=0.436]\u001b[A\n",
            "  9%|▉         | 10/110 [00:15<02:31,  1.52s/it, loss=0.436]\u001b[A\n",
            "  9%|▉         | 10/110 [00:16<02:31,  1.52s/it, loss=0.377]\u001b[A\n",
            " 10%|█         | 11/110 [00:16<02:31,  1.53s/it, loss=0.377]\u001b[A\n",
            " 10%|█         | 11/110 [00:18<02:31,  1.53s/it, loss=0.373]\u001b[A\n",
            " 11%|█         | 12/110 [00:18<02:29,  1.52s/it, loss=0.373]\u001b[A\n",
            " 11%|█         | 12/110 [00:19<02:29,  1.52s/it, loss=0.305]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:19<02:27,  1.52s/it, loss=0.305]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:21<02:27,  1.52s/it, loss=0.393]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:21<02:25,  1.52s/it, loss=0.393]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:22<02:25,  1.52s/it, loss=0.396]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:22<02:24,  1.52s/it, loss=0.396]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:24<02:24,  1.52s/it, loss=0.457]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:24<02:23,  1.53s/it, loss=0.457]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:25<02:23,  1.53s/it, loss=0.209]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:25<02:21,  1.53s/it, loss=0.209]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:27<02:21,  1.53s/it, loss=0.333]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:27<02:20,  1.52s/it, loss=0.333]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:28<02:20,  1.52s/it, loss=0.477]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:28<02:18,  1.52s/it, loss=0.477]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:30<02:18,  1.52s/it, loss=0.63] \u001b[A\n",
            " 18%|█▊        | 20/110 [00:30<02:16,  1.52s/it, loss=0.63]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:31<02:16,  1.52s/it, loss=0.439]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:32<02:16,  1.53s/it, loss=0.439]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:33<02:16,  1.53s/it, loss=0.273]\u001b[A\n",
            " 20%|██        | 22/110 [00:33<02:14,  1.53s/it, loss=0.273]\u001b[A\n",
            " 20%|██        | 22/110 [00:35<02:14,  1.53s/it, loss=0.414]\u001b[A\n",
            " 21%|██        | 23/110 [00:35<02:12,  1.53s/it, loss=0.414]\u001b[A\n",
            " 21%|██        | 23/110 [00:36<02:12,  1.53s/it, loss=0.595]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:36<02:10,  1.52s/it, loss=0.595]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:38<02:10,  1.52s/it, loss=0.567]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:38<02:09,  1.52s/it, loss=0.567]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:39<02:09,  1.52s/it, loss=0.214]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:39<02:08,  1.53s/it, loss=0.214]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:41<02:08,  1.53s/it, loss=0.363]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:41<02:06,  1.53s/it, loss=0.363]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:42<02:06,  1.53s/it, loss=0.564]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:42<02:04,  1.52s/it, loss=0.564]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:44<02:04,  1.52s/it, loss=0.369]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:44<02:03,  1.52s/it, loss=0.369]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:45<02:03,  1.52s/it, loss=0.438]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:45<02:01,  1.52s/it, loss=0.438]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:47<02:01,  1.52s/it, loss=0.446]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:47<02:01,  1.53s/it, loss=0.446]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:48<02:01,  1.53s/it, loss=0.276]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:48<01:59,  1.53s/it, loss=0.276]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:50<01:59,  1.53s/it, loss=0.374]\u001b[A\n",
            " 30%|███       | 33/110 [00:50<01:57,  1.53s/it, loss=0.374]\u001b[A\n",
            " 30%|███       | 33/110 [00:51<01:57,  1.53s/it, loss=0.337]\u001b[A\n",
            " 31%|███       | 34/110 [00:51<01:55,  1.52s/it, loss=0.337]\u001b[A\n",
            " 31%|███       | 34/110 [00:53<01:55,  1.52s/it, loss=0.513]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:53<01:54,  1.52s/it, loss=0.513]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:54<01:54,  1.52s/it, loss=0.373]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:54<01:53,  1.54s/it, loss=0.373]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:56<01:53,  1.54s/it, loss=0.307]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:56<01:51,  1.53s/it, loss=0.307]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:57<01:51,  1.53s/it, loss=0.397]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:57<01:49,  1.53s/it, loss=0.397]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:59<01:49,  1.53s/it, loss=0.486]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:59<01:48,  1.52s/it, loss=0.486]\u001b[A\n",
            " 35%|███▌      | 39/110 [01:00<01:48,  1.52s/it, loss=0.374]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:00<01:46,  1.52s/it, loss=0.374]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:02<01:46,  1.52s/it, loss=0.333]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:02<01:46,  1.54s/it, loss=0.333]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:04<01:46,  1.54s/it, loss=0.32] \u001b[A\n",
            " 38%|███▊      | 42/110 [01:04<01:44,  1.53s/it, loss=0.32]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:05<01:44,  1.53s/it, loss=0.472]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:05<01:42,  1.53s/it, loss=0.472]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:07<01:42,  1.53s/it, loss=0.286]\u001b[A\n",
            " 40%|████      | 44/110 [01:07<01:40,  1.52s/it, loss=0.286]\u001b[A\n",
            " 40%|████      | 44/110 [01:08<01:40,  1.52s/it, loss=0.41] \u001b[A\n",
            " 41%|████      | 45/110 [01:08<01:39,  1.52s/it, loss=0.41]\u001b[A\n",
            " 41%|████      | 45/110 [01:10<01:39,  1.52s/it, loss=0.31]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:10<01:38,  1.54s/it, loss=0.31]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:11<01:38,  1.54s/it, loss=0.547]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:11<01:36,  1.53s/it, loss=0.547]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:13<01:36,  1.53s/it, loss=0.355]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:13<01:34,  1.53s/it, loss=0.355]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:14<01:34,  1.53s/it, loss=0.333]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:14<01:33,  1.53s/it, loss=0.333]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:16<01:33,  1.53s/it, loss=0.465]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:16<01:31,  1.53s/it, loss=0.465]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:17<01:31,  1.53s/it, loss=0.51] \u001b[A\n",
            " 46%|████▋     | 51/110 [01:17<01:30,  1.54s/it, loss=0.51]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:19<01:30,  1.54s/it, loss=0.343]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:19<01:28,  1.53s/it, loss=0.343]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:20<01:28,  1.53s/it, loss=0.394]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:20<01:27,  1.53s/it, loss=0.394]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:22<01:27,  1.53s/it, loss=0.461]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:22<01:25,  1.52s/it, loss=0.461]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:23<01:25,  1.52s/it, loss=0.221]\u001b[A\n",
            " 50%|█████     | 55/110 [01:23<01:23,  1.52s/it, loss=0.221]\u001b[A\n",
            " 50%|█████     | 55/110 [01:25<01:23,  1.52s/it, loss=0.58] \u001b[A\n",
            " 51%|█████     | 56/110 [01:25<01:22,  1.54s/it, loss=0.58]\u001b[A\n",
            " 51%|█████     | 56/110 [01:26<01:22,  1.54s/it, loss=0.391]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:26<01:21,  1.53s/it, loss=0.391]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:28<01:21,  1.53s/it, loss=0.338]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:28<01:19,  1.53s/it, loss=0.338]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:30<01:19,  1.53s/it, loss=0.481]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:30<01:17,  1.52s/it, loss=0.481]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:31<01:17,  1.52s/it, loss=0.355]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:31<01:16,  1.52s/it, loss=0.355]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:33<01:16,  1.52s/it, loss=0.385]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:33<01:15,  1.54s/it, loss=0.385]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:34<01:15,  1.54s/it, loss=0.383]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:34<01:13,  1.53s/it, loss=0.383]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:36<01:13,  1.53s/it, loss=0.476]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:36<01:11,  1.53s/it, loss=0.476]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:37<01:11,  1.53s/it, loss=0.306]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:37<01:10,  1.52s/it, loss=0.306]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:39<01:10,  1.52s/it, loss=0.202]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:39<01:08,  1.52s/it, loss=0.202]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:40<01:08,  1.52s/it, loss=0.624]\u001b[A\n",
            " 60%|██████    | 66/110 [01:40<01:07,  1.54s/it, loss=0.624]\u001b[A\n",
            " 60%|██████    | 66/110 [01:42<01:07,  1.54s/it, loss=0.279]\u001b[A\n",
            " 61%|██████    | 67/110 [01:42<01:05,  1.53s/it, loss=0.279]\u001b[A\n",
            " 61%|██████    | 67/110 [01:43<01:05,  1.53s/it, loss=0.345]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:43<01:04,  1.53s/it, loss=0.345]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:45<01:04,  1.53s/it, loss=0.38] \u001b[A\n",
            " 63%|██████▎   | 69/110 [01:45<01:02,  1.53s/it, loss=0.38]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:46<01:02,  1.53s/it, loss=0.355]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:46<01:00,  1.52s/it, loss=0.355]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:48<01:00,  1.52s/it, loss=0.266]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:48<00:59,  1.54s/it, loss=0.266]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:49<00:59,  1.54s/it, loss=0.348]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:49<00:58,  1.53s/it, loss=0.348]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:51<00:58,  1.53s/it, loss=0.299]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:51<00:56,  1.53s/it, loss=0.299]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:52<00:56,  1.53s/it, loss=0.523]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:52<00:54,  1.52s/it, loss=0.523]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:54<00:54,  1.52s/it, loss=0.373]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:54<00:53,  1.52s/it, loss=0.373]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:56<00:53,  1.52s/it, loss=0.497]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:56<00:52,  1.54s/it, loss=0.497]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:57<00:52,  1.54s/it, loss=0.305]\u001b[A\n",
            " 70%|███████   | 77/110 [01:57<00:50,  1.53s/it, loss=0.305]\u001b[A\n",
            " 70%|███████   | 77/110 [01:59<00:50,  1.53s/it, loss=0.316]\u001b[A\n",
            " 71%|███████   | 78/110 [01:59<00:48,  1.53s/it, loss=0.316]\u001b[A\n",
            " 71%|███████   | 78/110 [02:00<00:48,  1.53s/it, loss=0.555]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:00<00:47,  1.52s/it, loss=0.555]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:02<00:47,  1.52s/it, loss=0.297]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:02<00:45,  1.52s/it, loss=0.297]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:03<00:45,  1.52s/it, loss=0.357]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:03<00:44,  1.54s/it, loss=0.357]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:05<00:44,  1.54s/it, loss=0.369]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:05<00:42,  1.53s/it, loss=0.369]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:06<00:42,  1.53s/it, loss=0.416]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:06<00:41,  1.53s/it, loss=0.416]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:08<00:41,  1.53s/it, loss=0.559]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:08<00:39,  1.53s/it, loss=0.559]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:09<00:39,  1.53s/it, loss=0.392]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:09<00:38,  1.52s/it, loss=0.392]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:11<00:38,  1.52s/it, loss=0.483]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:11<00:36,  1.54s/it, loss=0.483]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:12<00:36,  1.54s/it, loss=0.315]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:12<00:35,  1.53s/it, loss=0.315]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:14<00:35,  1.53s/it, loss=0.326]\u001b[A\n",
            " 80%|████████  | 88/110 [02:14<00:33,  1.53s/it, loss=0.326]\u001b[A\n",
            " 80%|████████  | 88/110 [02:15<00:33,  1.53s/it, loss=0.391]\u001b[A\n",
            " 81%|████████  | 89/110 [02:15<00:32,  1.53s/it, loss=0.391]\u001b[A\n",
            " 81%|████████  | 89/110 [02:17<00:32,  1.53s/it, loss=0.36] \u001b[A\n",
            " 82%|████████▏ | 90/110 [02:17<00:30,  1.53s/it, loss=0.36]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:19<00:30,  1.53s/it, loss=0.491]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:19<00:29,  1.54s/it, loss=0.491]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:20<00:29,  1.54s/it, loss=0.312]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:20<00:27,  1.53s/it, loss=0.312]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:22<00:27,  1.53s/it, loss=0.452]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:22<00:25,  1.53s/it, loss=0.452]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:23<00:25,  1.53s/it, loss=0.515]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:23<00:24,  1.52s/it, loss=0.515]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:25<00:24,  1.52s/it, loss=0.31] \u001b[A\n",
            " 86%|████████▋ | 95/110 [02:25<00:22,  1.52s/it, loss=0.31]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:26<00:22,  1.52s/it, loss=0.325]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:26<00:21,  1.54s/it, loss=0.325]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:28<00:21,  1.54s/it, loss=0.509]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:28<00:19,  1.53s/it, loss=0.509]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:29<00:19,  1.53s/it, loss=0.426]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:29<00:18,  1.53s/it, loss=0.426]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:31<00:18,  1.53s/it, loss=0.405]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:31<00:16,  1.52s/it, loss=0.405]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:32<00:16,  1.52s/it, loss=0.299]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:32<00:15,  1.52s/it, loss=0.299]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:34<00:15,  1.52s/it, loss=0.255]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:34<00:13,  1.54s/it, loss=0.255]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:35<00:13,  1.54s/it, loss=0.408]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:35<00:12,  1.53s/it, loss=0.408]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:37<00:12,  1.53s/it, loss=0.362]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:37<00:10,  1.53s/it, loss=0.362]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:38<00:10,  1.53s/it, loss=0.262]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:38<00:09,  1.52s/it, loss=0.262]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:40<00:09,  1.52s/it, loss=0.433]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:40<00:07,  1.52s/it, loss=0.433]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:41<00:07,  1.52s/it, loss=0.246]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:41<00:06,  1.53s/it, loss=0.246]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:43<00:06,  1.53s/it, loss=0.337]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:43<00:04,  1.53s/it, loss=0.337]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:44<00:04,  1.53s/it, loss=0.328]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:44<00:03,  1.53s/it, loss=0.328]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:46<00:03,  1.53s/it, loss=0.359]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.52s/it, loss=0.359]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.52s/it, loss=0.389]\u001b[A\n",
            "100%|██████████| 110/110 [02:46<00:00,  1.21s/it, loss=0.389]\u001b[A\n",
            "                                                             \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:13,  1.93it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:13,  1.91it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:02<00:12,  1.90it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:12,  1.89it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:03<00:11,  1.89it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:11,  1.89it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:04<00:10,  1.89it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:10,  1.89it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:05<00:09,  1.90it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  1.89it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:06<00:08,  1.90it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  1.89it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:07<00:07,  1.90it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  1.89it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:08<00:06,  1.89it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  1.89it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:09<00:05,  1.89it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:10<00:04,  1.89it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:10<00:04,  1.89it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:11<00:03,  1.89it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:11<00:03,  1.89it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:12<00:02,  1.88it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:12<00:02,  1.89it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:13<00:01,  1.89it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:13<00:01,  1.89it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:14<00:00,  1.89it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:14<00:00,  2.33it/s]\u001b[A\n",
            " 50%|█████     | 3/6 [07:09<06:15, 125.26s/it]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.8803, F1 score @0.5 = 0.3770\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/110 [00:01<?, ?it/s, loss=0.333]\u001b[A\n",
            "  1%|          | 1/110 [00:01<02:49,  1.56s/it, loss=0.333]\u001b[A\n",
            "  1%|          | 1/110 [00:03<02:49,  1.56s/it, loss=0.424]\u001b[A\n",
            "  2%|▏         | 2/110 [00:03<02:47,  1.55s/it, loss=0.424]\u001b[A\n",
            "  2%|▏         | 2/110 [00:04<02:47,  1.55s/it, loss=0.323]\u001b[A\n",
            "  3%|▎         | 3/110 [00:04<02:44,  1.54s/it, loss=0.323]\u001b[A\n",
            "  3%|▎         | 3/110 [00:06<02:44,  1.54s/it, loss=0.254]\u001b[A\n",
            "  4%|▎         | 4/110 [00:06<02:42,  1.53s/it, loss=0.254]\u001b[A\n",
            "  4%|▎         | 4/110 [00:07<02:42,  1.53s/it, loss=0.297]\u001b[A\n",
            "  5%|▍         | 5/110 [00:07<02:40,  1.53s/it, loss=0.297]\u001b[A\n",
            "  5%|▍         | 5/110 [00:09<02:40,  1.53s/it, loss=0.18] \u001b[A\n",
            "  5%|▌         | 6/110 [00:09<02:40,  1.54s/it, loss=0.18]\u001b[A\n",
            "  5%|▌         | 6/110 [00:10<02:40,  1.54s/it, loss=0.326]\u001b[A\n",
            "  6%|▋         | 7/110 [00:10<02:38,  1.53s/it, loss=0.326]\u001b[A\n",
            "  6%|▋         | 7/110 [00:12<02:38,  1.53s/it, loss=0.312]\u001b[A\n",
            "  7%|▋         | 8/110 [00:12<02:36,  1.53s/it, loss=0.312]\u001b[A\n",
            "  7%|▋         | 8/110 [00:13<02:36,  1.53s/it, loss=0.289]\u001b[A\n",
            "  8%|▊         | 9/110 [00:13<02:34,  1.53s/it, loss=0.289]\u001b[A\n",
            "  8%|▊         | 9/110 [00:15<02:34,  1.53s/it, loss=0.184]\u001b[A\n",
            "  9%|▉         | 10/110 [00:15<02:32,  1.53s/it, loss=0.184]\u001b[A\n",
            "  9%|▉         | 10/110 [00:16<02:32,  1.53s/it, loss=0.295]\u001b[A\n",
            " 10%|█         | 11/110 [00:16<02:32,  1.54s/it, loss=0.295]\u001b[A\n",
            " 10%|█         | 11/110 [00:18<02:32,  1.54s/it, loss=0.231]\u001b[A\n",
            " 11%|█         | 12/110 [00:18<02:30,  1.53s/it, loss=0.231]\u001b[A\n",
            " 11%|█         | 12/110 [00:19<02:30,  1.53s/it, loss=0.348]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:19<02:28,  1.53s/it, loss=0.348]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:21<02:28,  1.53s/it, loss=0.422]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:21<02:26,  1.52s/it, loss=0.422]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:22<02:26,  1.52s/it, loss=0.302]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:22<02:24,  1.52s/it, loss=0.302]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:24<02:24,  1.52s/it, loss=0.377]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:24<02:24,  1.54s/it, loss=0.377]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:26<02:24,  1.54s/it, loss=0.459]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:26<02:22,  1.53s/it, loss=0.459]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:27<02:22,  1.53s/it, loss=0.222]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:27<02:20,  1.53s/it, loss=0.222]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:29<02:20,  1.53s/it, loss=0.351]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:29<02:18,  1.53s/it, loss=0.351]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:30<02:18,  1.53s/it, loss=0.424]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:30<02:17,  1.53s/it, loss=0.424]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:32<02:17,  1.53s/it, loss=0.156]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:32<02:16,  1.54s/it, loss=0.156]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:33<02:16,  1.54s/it, loss=0.176]\u001b[A\n",
            " 20%|██        | 22/110 [00:33<02:14,  1.53s/it, loss=0.176]\u001b[A\n",
            " 20%|██        | 22/110 [00:35<02:14,  1.53s/it, loss=0.184]\u001b[A\n",
            " 21%|██        | 23/110 [00:35<02:12,  1.53s/it, loss=0.184]\u001b[A\n",
            " 21%|██        | 23/110 [00:36<02:12,  1.53s/it, loss=0.195]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:36<02:11,  1.53s/it, loss=0.195]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:38<02:11,  1.53s/it, loss=0.41] \u001b[A\n",
            " 23%|██▎       | 25/110 [00:38<02:09,  1.52s/it, loss=0.41]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:39<02:09,  1.52s/it, loss=0.277]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:39<02:09,  1.54s/it, loss=0.277]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:41<02:09,  1.54s/it, loss=0.446]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:41<02:07,  1.53s/it, loss=0.446]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:42<02:07,  1.53s/it, loss=0.198]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:42<02:05,  1.53s/it, loss=0.198]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:44<02:05,  1.53s/it, loss=0.271]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:44<02:03,  1.52s/it, loss=0.271]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:45<02:03,  1.52s/it, loss=0.284]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:45<02:02,  1.53s/it, loss=0.284]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:47<02:02,  1.53s/it, loss=0.393]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:47<02:01,  1.54s/it, loss=0.393]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:48<02:01,  1.54s/it, loss=0.346]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:48<01:59,  1.53s/it, loss=0.346]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:50<01:59,  1.53s/it, loss=0.387]\u001b[A\n",
            " 30%|███       | 33/110 [00:50<01:57,  1.53s/it, loss=0.387]\u001b[A\n",
            " 30%|███       | 33/110 [00:52<01:57,  1.53s/it, loss=0.327]\u001b[A\n",
            " 31%|███       | 34/110 [00:52<01:55,  1.52s/it, loss=0.327]\u001b[A\n",
            " 31%|███       | 34/110 [00:53<01:55,  1.52s/it, loss=0.314]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:53<01:54,  1.52s/it, loss=0.314]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:55<01:54,  1.52s/it, loss=0.245]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:55<01:53,  1.54s/it, loss=0.245]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:56<01:53,  1.54s/it, loss=0.395]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:56<01:51,  1.53s/it, loss=0.395]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:58<01:51,  1.53s/it, loss=0.188]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:58<01:50,  1.53s/it, loss=0.188]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:59<01:50,  1.53s/it, loss=0.337]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:59<01:48,  1.53s/it, loss=0.337]\u001b[A\n",
            " 35%|███▌      | 39/110 [01:01<01:48,  1.53s/it, loss=0.26] \u001b[A\n",
            " 36%|███▋      | 40/110 [01:01<01:46,  1.52s/it, loss=0.26]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:02<01:46,  1.52s/it, loss=0.344]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:02<01:46,  1.54s/it, loss=0.344]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:04<01:46,  1.54s/it, loss=0.17] \u001b[A\n",
            " 38%|███▊      | 42/110 [01:04<01:44,  1.53s/it, loss=0.17]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:05<01:44,  1.53s/it, loss=0.335]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:05<01:42,  1.53s/it, loss=0.335]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:07<01:42,  1.53s/it, loss=0.358]\u001b[A\n",
            " 40%|████      | 44/110 [01:07<01:40,  1.53s/it, loss=0.358]\u001b[A\n",
            " 40%|████      | 44/110 [01:08<01:40,  1.53s/it, loss=0.308]\u001b[A\n",
            " 41%|████      | 45/110 [01:08<01:39,  1.53s/it, loss=0.308]\u001b[A\n",
            " 41%|████      | 45/110 [01:10<01:39,  1.53s/it, loss=0.281]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:10<01:38,  1.54s/it, loss=0.281]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:11<01:38,  1.54s/it, loss=0.342]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:11<01:36,  1.53s/it, loss=0.342]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:13<01:36,  1.53s/it, loss=0.159]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:13<01:34,  1.53s/it, loss=0.159]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:14<01:34,  1.53s/it, loss=0.266]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:14<01:32,  1.52s/it, loss=0.266]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:16<01:32,  1.52s/it, loss=0.27] \u001b[A\n",
            " 45%|████▌     | 50/110 [01:16<01:31,  1.52s/it, loss=0.27]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:18<01:31,  1.52s/it, loss=0.219]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:18<01:30,  1.54s/it, loss=0.219]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:19<01:30,  1.54s/it, loss=0.334]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:19<01:28,  1.53s/it, loss=0.334]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:21<01:28,  1.53s/it, loss=0.321]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:21<01:27,  1.53s/it, loss=0.321]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:22<01:27,  1.53s/it, loss=0.348]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:22<01:25,  1.53s/it, loss=0.348]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:24<01:25,  1.53s/it, loss=0.328]\u001b[A\n",
            " 50%|█████     | 55/110 [01:24<01:23,  1.52s/it, loss=0.328]\u001b[A\n",
            " 50%|█████     | 55/110 [01:25<01:23,  1.52s/it, loss=0.221]\u001b[A\n",
            " 51%|█████     | 56/110 [01:25<01:22,  1.54s/it, loss=0.221]\u001b[A\n",
            " 51%|█████     | 56/110 [01:27<01:22,  1.54s/it, loss=0.297]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:27<01:21,  1.53s/it, loss=0.297]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:28<01:21,  1.53s/it, loss=0.183]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:28<01:19,  1.53s/it, loss=0.183]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:30<01:19,  1.53s/it, loss=0.13] \u001b[A\n",
            " 54%|█████▎    | 59/110 [01:30<01:17,  1.52s/it, loss=0.13]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:31<01:17,  1.52s/it, loss=0.354]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:31<01:16,  1.52s/it, loss=0.354]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:33<01:16,  1.52s/it, loss=0.15] \u001b[A\n",
            " 55%|█████▌    | 61/110 [01:33<01:15,  1.54s/it, loss=0.15]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:34<01:15,  1.54s/it, loss=0.448]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:34<01:13,  1.53s/it, loss=0.448]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:36<01:13,  1.53s/it, loss=0.547]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:36<01:11,  1.53s/it, loss=0.547]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:37<01:11,  1.53s/it, loss=0.419]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:37<01:10,  1.52s/it, loss=0.419]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:39<01:10,  1.52s/it, loss=0.151]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:39<01:08,  1.52s/it, loss=0.151]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:40<01:08,  1.52s/it, loss=0.24] \u001b[A\n",
            " 60%|██████    | 66/110 [01:40<01:07,  1.53s/it, loss=0.24]\u001b[A\n",
            " 60%|██████    | 66/110 [01:42<01:07,  1.53s/it, loss=0.394]\u001b[A\n",
            " 61%|██████    | 67/110 [01:42<01:05,  1.53s/it, loss=0.394]\u001b[A\n",
            " 61%|██████    | 67/110 [01:43<01:05,  1.53s/it, loss=0.343]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:43<01:03,  1.52s/it, loss=0.343]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:45<01:03,  1.52s/it, loss=0.31] \u001b[A\n",
            " 63%|██████▎   | 69/110 [01:45<01:02,  1.52s/it, loss=0.31]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:47<01:02,  1.52s/it, loss=0.359]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:47<01:00,  1.52s/it, loss=0.359]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:48<01:00,  1.52s/it, loss=0.5]  \u001b[A\n",
            " 65%|██████▍   | 71/110 [01:48<00:59,  1.53s/it, loss=0.5]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:50<00:59,  1.53s/it, loss=0.333]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:50<00:58,  1.53s/it, loss=0.333]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:51<00:58,  1.53s/it, loss=0.394]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:51<00:56,  1.52s/it, loss=0.394]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:53<00:56,  1.52s/it, loss=0.361]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:53<00:54,  1.52s/it, loss=0.361]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:54<00:54,  1.52s/it, loss=0.358]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:54<00:53,  1.52s/it, loss=0.358]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:56<00:53,  1.52s/it, loss=0.438]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:56<00:52,  1.53s/it, loss=0.438]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:57<00:52,  1.53s/it, loss=0.264]\u001b[A\n",
            " 70%|███████   | 77/110 [01:57<00:50,  1.52s/it, loss=0.264]\u001b[A\n",
            " 70%|███████   | 77/110 [01:59<00:50,  1.52s/it, loss=0.392]\u001b[A\n",
            " 71%|███████   | 78/110 [01:59<00:48,  1.52s/it, loss=0.392]\u001b[A\n",
            " 71%|███████   | 78/110 [02:00<00:48,  1.52s/it, loss=0.291]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:00<00:47,  1.52s/it, loss=0.291]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:02<00:47,  1.52s/it, loss=0.431]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:02<00:45,  1.52s/it, loss=0.431]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:03<00:45,  1.52s/it, loss=0.386]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:03<00:44,  1.53s/it, loss=0.386]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:05<00:44,  1.53s/it, loss=0.296]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:05<00:42,  1.52s/it, loss=0.296]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:06<00:42,  1.52s/it, loss=0.397]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:06<00:41,  1.52s/it, loss=0.397]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:08<00:41,  1.52s/it, loss=0.345]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:08<00:39,  1.52s/it, loss=0.345]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:09<00:39,  1.52s/it, loss=0.261]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:09<00:38,  1.52s/it, loss=0.261]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:11<00:38,  1.52s/it, loss=0.332]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:11<00:36,  1.53s/it, loss=0.332]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:12<00:36,  1.53s/it, loss=0.283]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:12<00:35,  1.53s/it, loss=0.283]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:14<00:35,  1.53s/it, loss=0.261]\u001b[A\n",
            " 80%|████████  | 88/110 [02:14<00:33,  1.52s/it, loss=0.261]\u001b[A\n",
            " 80%|████████  | 88/110 [02:15<00:33,  1.52s/it, loss=0.335]\u001b[A\n",
            " 81%|████████  | 89/110 [02:15<00:31,  1.52s/it, loss=0.335]\u001b[A\n",
            " 81%|████████  | 89/110 [02:17<00:31,  1.52s/it, loss=0.265]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:17<00:30,  1.52s/it, loss=0.265]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:19<00:30,  1.52s/it, loss=0.181]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:19<00:29,  1.53s/it, loss=0.181]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:20<00:29,  1.53s/it, loss=0.427]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:20<00:27,  1.52s/it, loss=0.427]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:22<00:27,  1.52s/it, loss=0.299]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:22<00:25,  1.52s/it, loss=0.299]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:23<00:25,  1.52s/it, loss=0.38] \u001b[A\n",
            " 85%|████████▌ | 94/110 [02:23<00:24,  1.52s/it, loss=0.38]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:25<00:24,  1.52s/it, loss=0.156]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:25<00:22,  1.51s/it, loss=0.156]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:26<00:22,  1.51s/it, loss=0.292]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:26<00:21,  1.53s/it, loss=0.292]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:28<00:21,  1.53s/it, loss=0.453]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:28<00:19,  1.52s/it, loss=0.453]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:29<00:19,  1.52s/it, loss=0.322]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:29<00:18,  1.52s/it, loss=0.322]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:31<00:18,  1.52s/it, loss=0.135]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:31<00:16,  1.51s/it, loss=0.135]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:32<00:16,  1.51s/it, loss=0.173]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:32<00:15,  1.52s/it, loss=0.173]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:34<00:15,  1.52s/it, loss=0.164]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:34<00:13,  1.53s/it, loss=0.164]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:35<00:13,  1.53s/it, loss=0.19] \u001b[A\n",
            " 93%|█████████▎| 102/110 [02:35<00:12,  1.52s/it, loss=0.19]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:37<00:12,  1.52s/it, loss=0.267]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:37<00:10,  1.52s/it, loss=0.267]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:38<00:10,  1.52s/it, loss=0.428]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:38<00:09,  1.51s/it, loss=0.428]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:40<00:09,  1.51s/it, loss=0.185]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:40<00:07,  1.52s/it, loss=0.185]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:41<00:07,  1.52s/it, loss=0.26] \u001b[A\n",
            " 96%|█████████▋| 106/110 [02:41<00:06,  1.53s/it, loss=0.26]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:43<00:06,  1.53s/it, loss=0.186]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:43<00:04,  1.52s/it, loss=0.186]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:44<00:04,  1.52s/it, loss=0.317]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:44<00:03,  1.52s/it, loss=0.317]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:46<00:03,  1.52s/it, loss=0.202]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.52s/it, loss=0.202]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.52s/it, loss=0.127]\u001b[A\n",
            "100%|██████████| 110/110 [02:46<00:00,  1.21s/it, loss=0.127]\u001b[A\n",
            "                                                             \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:13,  1.94it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:01<00:13,  1.93it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:11,  1.92it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:03<00:11,  1.91it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:10,  1.92it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:04<00:10,  1.91it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:09,  1.91it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:05<00:09,  1.91it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  1.91it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:06<00:08,  1.91it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  1.91it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:07<00:07,  1.91it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  1.90it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:08<00:06,  1.91it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  1.90it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:09<00:05,  1.90it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:09<00:04,  1.90it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:10<00:04,  1.91it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:11<00:03,  1.91it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:11<00:03,  1.91it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:12<00:02,  1.91it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:12<00:02,  1.91it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:14<00:00,  1.90it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:14<00:00,  2.36it/s]\u001b[A\n",
            " 67%|██████▋   | 4/6 [10:10<04:44, 142.05s/it]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.9201, F1 score @0.5 = 0.6875\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/110 [00:01<?, ?it/s, loss=0.245]\u001b[A\n",
            "  1%|          | 1/110 [00:01<02:48,  1.54s/it, loss=0.245]\u001b[A\n",
            "  1%|          | 1/110 [00:03<02:48,  1.54s/it, loss=0.354]\u001b[A\n",
            "  2%|▏         | 2/110 [00:03<02:45,  1.53s/it, loss=0.354]\u001b[A\n",
            "  2%|▏         | 2/110 [00:04<02:45,  1.53s/it, loss=0.243]\u001b[A\n",
            "  3%|▎         | 3/110 [00:04<02:43,  1.53s/it, loss=0.243]\u001b[A\n",
            "  3%|▎         | 3/110 [00:06<02:43,  1.53s/it, loss=0.359]\u001b[A\n",
            "  4%|▎         | 4/110 [00:06<02:41,  1.52s/it, loss=0.359]\u001b[A\n",
            "  4%|▎         | 4/110 [00:07<02:41,  1.52s/it, loss=0.228]\u001b[A\n",
            "  5%|▍         | 5/110 [00:07<02:39,  1.52s/it, loss=0.228]\u001b[A\n",
            "  5%|▍         | 5/110 [00:09<02:39,  1.52s/it, loss=0.169]\u001b[A\n",
            "  5%|▌         | 6/110 [00:09<02:39,  1.53s/it, loss=0.169]\u001b[A\n",
            "  5%|▌         | 6/110 [00:10<02:39,  1.53s/it, loss=0.346]\u001b[A\n",
            "  6%|▋         | 7/110 [00:10<02:36,  1.52s/it, loss=0.346]\u001b[A\n",
            "  6%|▋         | 7/110 [00:12<02:36,  1.52s/it, loss=0.214]\u001b[A\n",
            "  7%|▋         | 8/110 [00:12<02:34,  1.52s/it, loss=0.214]\u001b[A\n",
            "  7%|▋         | 8/110 [00:13<02:34,  1.52s/it, loss=0.235]\u001b[A\n",
            "  8%|▊         | 9/110 [00:13<02:32,  1.51s/it, loss=0.235]\u001b[A\n",
            "  8%|▊         | 9/110 [00:15<02:32,  1.51s/it, loss=0.303]\u001b[A\n",
            "  9%|▉         | 10/110 [00:15<02:31,  1.51s/it, loss=0.303]\u001b[A\n",
            "  9%|▉         | 10/110 [00:16<02:31,  1.51s/it, loss=0.186]\u001b[A\n",
            " 10%|█         | 11/110 [00:16<02:30,  1.52s/it, loss=0.186]\u001b[A\n",
            " 10%|█         | 11/110 [00:18<02:30,  1.52s/it, loss=0.258]\u001b[A\n",
            " 11%|█         | 12/110 [00:18<02:28,  1.52s/it, loss=0.258]\u001b[A\n",
            " 11%|█         | 12/110 [00:19<02:28,  1.52s/it, loss=0.276]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:19<02:27,  1.52s/it, loss=0.276]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:21<02:27,  1.52s/it, loss=0.249]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:21<02:25,  1.51s/it, loss=0.249]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:22<02:25,  1.51s/it, loss=0.24] \u001b[A\n",
            " 14%|█▎        | 15/110 [00:22<02:23,  1.51s/it, loss=0.24]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:24<02:23,  1.51s/it, loss=0.377]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:24<02:23,  1.53s/it, loss=0.377]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:25<02:23,  1.53s/it, loss=0.233]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:25<02:21,  1.52s/it, loss=0.233]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:27<02:21,  1.52s/it, loss=0.406]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:27<02:19,  1.52s/it, loss=0.406]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:28<02:19,  1.52s/it, loss=0.264]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:28<02:17,  1.52s/it, loss=0.264]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:30<02:17,  1.52s/it, loss=0.369]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:30<02:16,  1.51s/it, loss=0.369]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:31<02:16,  1.51s/it, loss=0.171]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:31<02:15,  1.53s/it, loss=0.171]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:33<02:15,  1.53s/it, loss=0.178]\u001b[A\n",
            " 20%|██        | 22/110 [00:33<02:13,  1.52s/it, loss=0.178]\u001b[A\n",
            " 20%|██        | 22/110 [00:34<02:13,  1.52s/it, loss=0.454]\u001b[A\n",
            " 21%|██        | 23/110 [00:34<02:12,  1.52s/it, loss=0.454]\u001b[A\n",
            " 21%|██        | 23/110 [00:36<02:12,  1.52s/it, loss=0.469]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:36<02:10,  1.52s/it, loss=0.469]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:37<02:10,  1.52s/it, loss=0.198]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:37<02:08,  1.52s/it, loss=0.198]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:39<02:08,  1.52s/it, loss=0.264]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:39<02:08,  1.53s/it, loss=0.264]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:41<02:08,  1.53s/it, loss=0.32] \u001b[A\n",
            " 25%|██▍       | 27/110 [00:41<02:06,  1.53s/it, loss=0.32]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:42<02:06,  1.53s/it, loss=0.215]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:42<02:04,  1.52s/it, loss=0.215]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:44<02:04,  1.52s/it, loss=0.241]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:44<02:03,  1.52s/it, loss=0.241]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:45<02:03,  1.52s/it, loss=0.2]  \u001b[A\n",
            " 27%|██▋       | 30/110 [00:45<02:01,  1.52s/it, loss=0.2]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:47<02:01,  1.52s/it, loss=0.282]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:47<02:01,  1.53s/it, loss=0.282]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:48<02:01,  1.53s/it, loss=0.285]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:48<01:59,  1.53s/it, loss=0.285]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:50<01:59,  1.53s/it, loss=0.215]\u001b[A\n",
            " 30%|███       | 33/110 [00:50<01:57,  1.53s/it, loss=0.215]\u001b[A\n",
            " 30%|███       | 33/110 [00:51<01:57,  1.53s/it, loss=0.204]\u001b[A\n",
            " 31%|███       | 34/110 [00:51<01:55,  1.52s/it, loss=0.204]\u001b[A\n",
            " 31%|███       | 34/110 [00:53<01:55,  1.52s/it, loss=0.328]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:53<01:54,  1.52s/it, loss=0.328]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:54<01:54,  1.52s/it, loss=0.292]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:54<01:53,  1.53s/it, loss=0.292]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:56<01:53,  1.53s/it, loss=0.228]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:56<01:51,  1.53s/it, loss=0.228]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:57<01:51,  1.53s/it, loss=0.191]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:57<01:49,  1.52s/it, loss=0.191]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:59<01:49,  1.52s/it, loss=0.27] \u001b[A\n",
            " 35%|███▌      | 39/110 [00:59<01:48,  1.52s/it, loss=0.27]\u001b[A\n",
            " 35%|███▌      | 39/110 [01:00<01:48,  1.52s/it, loss=0.342]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:00<01:46,  1.52s/it, loss=0.342]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:02<01:46,  1.52s/it, loss=0.125]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:02<01:45,  1.53s/it, loss=0.125]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:03<01:45,  1.53s/it, loss=0.185]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:03<01:43,  1.53s/it, loss=0.185]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:05<01:43,  1.53s/it, loss=0.159]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:05<01:42,  1.53s/it, loss=0.159]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:06<01:42,  1.53s/it, loss=0.214]\u001b[A\n",
            " 40%|████      | 44/110 [01:06<01:40,  1.52s/it, loss=0.214]\u001b[A\n",
            " 40%|████      | 44/110 [01:08<01:40,  1.52s/it, loss=0.3]  \u001b[A\n",
            " 41%|████      | 45/110 [01:08<01:39,  1.52s/it, loss=0.3]\u001b[A\n",
            " 41%|████      | 45/110 [01:10<01:39,  1.52s/it, loss=0.295]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:10<01:38,  1.54s/it, loss=0.295]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:11<01:38,  1.54s/it, loss=0.131]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:11<01:36,  1.53s/it, loss=0.131]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:13<01:36,  1.53s/it, loss=0.334]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:13<01:34,  1.53s/it, loss=0.334]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:14<01:34,  1.53s/it, loss=0.183]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:14<01:32,  1.52s/it, loss=0.183]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:16<01:32,  1.52s/it, loss=0.128]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:16<01:31,  1.52s/it, loss=0.128]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:17<01:31,  1.52s/it, loss=0.0912]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:17<01:30,  1.54s/it, loss=0.0912]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:19<01:30,  1.54s/it, loss=0.224] \u001b[A\n",
            " 47%|████▋     | 52/110 [01:19<01:28,  1.53s/it, loss=0.224]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:20<01:28,  1.53s/it, loss=0.384]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:20<01:26,  1.52s/it, loss=0.384]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:22<01:26,  1.52s/it, loss=0.0759]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:22<01:25,  1.52s/it, loss=0.0759]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:23<01:25,  1.52s/it, loss=0.131] \u001b[A\n",
            " 50%|█████     | 55/110 [01:23<01:23,  1.52s/it, loss=0.131]\u001b[A\n",
            " 50%|█████     | 55/110 [01:25<01:23,  1.52s/it, loss=0.165]\u001b[A\n",
            " 51%|█████     | 56/110 [01:25<01:22,  1.54s/it, loss=0.165]\u001b[A\n",
            " 51%|█████     | 56/110 [01:26<01:22,  1.54s/it, loss=0.215]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:26<01:21,  1.53s/it, loss=0.215]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:28<01:21,  1.53s/it, loss=0.305]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:28<01:19,  1.53s/it, loss=0.305]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:29<01:19,  1.53s/it, loss=0.0622]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:29<01:17,  1.52s/it, loss=0.0622]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:31<01:17,  1.52s/it, loss=0.0876]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:31<01:16,  1.52s/it, loss=0.0876]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:32<01:16,  1.52s/it, loss=0.2]   \u001b[A\n",
            " 55%|█████▌    | 61/110 [01:32<01:15,  1.54s/it, loss=0.2]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:34<01:15,  1.54s/it, loss=0.137]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:34<01:13,  1.53s/it, loss=0.137]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:35<01:13,  1.53s/it, loss=0.153]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:35<01:11,  1.53s/it, loss=0.153]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:37<01:11,  1.53s/it, loss=0.266]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:37<01:10,  1.52s/it, loss=0.266]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:39<01:10,  1.52s/it, loss=0.119]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:39<01:08,  1.52s/it, loss=0.119]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:40<01:08,  1.52s/it, loss=0.235]\u001b[A\n",
            " 60%|██████    | 66/110 [01:40<01:07,  1.54s/it, loss=0.235]\u001b[A\n",
            " 60%|██████    | 66/110 [01:42<01:07,  1.54s/it, loss=0.145]\u001b[A\n",
            " 61%|██████    | 67/110 [01:42<01:05,  1.53s/it, loss=0.145]\u001b[A\n",
            " 61%|██████    | 67/110 [01:43<01:05,  1.53s/it, loss=0.142]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:43<01:04,  1.53s/it, loss=0.142]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:45<01:04,  1.53s/it, loss=0.195]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:45<01:02,  1.52s/it, loss=0.195]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:46<01:02,  1.52s/it, loss=0.137]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:46<01:00,  1.52s/it, loss=0.137]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:48<01:00,  1.52s/it, loss=0.165]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:48<00:59,  1.53s/it, loss=0.165]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:49<00:59,  1.53s/it, loss=0.116]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:49<00:58,  1.53s/it, loss=0.116]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:51<00:58,  1.53s/it, loss=0.261]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:51<00:56,  1.53s/it, loss=0.261]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:52<00:56,  1.53s/it, loss=0.251]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:52<00:54,  1.52s/it, loss=0.251]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:54<00:54,  1.52s/it, loss=0.281]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:54<00:53,  1.52s/it, loss=0.281]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:55<00:53,  1.52s/it, loss=0.107]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:55<00:52,  1.54s/it, loss=0.107]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:57<00:52,  1.54s/it, loss=0.19] \u001b[A\n",
            " 70%|███████   | 77/110 [01:57<00:50,  1.53s/it, loss=0.19]\u001b[A\n",
            " 70%|███████   | 77/110 [01:58<00:50,  1.53s/it, loss=0.332]\u001b[A\n",
            " 71%|███████   | 78/110 [01:58<00:48,  1.53s/it, loss=0.332]\u001b[A\n",
            " 71%|███████   | 78/110 [02:00<00:48,  1.53s/it, loss=0.193]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:00<00:47,  1.52s/it, loss=0.193]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:01<00:47,  1.52s/it, loss=0.104]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:01<00:45,  1.52s/it, loss=0.104]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:03<00:45,  1.52s/it, loss=0.549]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:03<00:44,  1.54s/it, loss=0.549]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:05<00:44,  1.54s/it, loss=0.114]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:05<00:42,  1.53s/it, loss=0.114]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:06<00:42,  1.53s/it, loss=0.33] \u001b[A\n",
            " 75%|███████▌  | 83/110 [02:06<00:41,  1.53s/it, loss=0.33]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:08<00:41,  1.53s/it, loss=0.0714]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:08<00:39,  1.53s/it, loss=0.0714]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:09<00:39,  1.53s/it, loss=0.0912]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:09<00:38,  1.52s/it, loss=0.0912]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:11<00:38,  1.52s/it, loss=0.0668]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:11<00:36,  1.54s/it, loss=0.0668]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:12<00:36,  1.54s/it, loss=0.0485]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:12<00:35,  1.53s/it, loss=0.0485]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:14<00:35,  1.53s/it, loss=0.0996]\u001b[A\n",
            " 80%|████████  | 88/110 [02:14<00:33,  1.53s/it, loss=0.0996]\u001b[A\n",
            " 80%|████████  | 88/110 [02:15<00:33,  1.53s/it, loss=0.14]  \u001b[A\n",
            " 81%|████████  | 89/110 [02:15<00:32,  1.53s/it, loss=0.14]\u001b[A\n",
            " 81%|████████  | 89/110 [02:17<00:32,  1.53s/it, loss=0.16]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:17<00:30,  1.52s/it, loss=0.16]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:18<00:30,  1.52s/it, loss=0.0633]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:18<00:29,  1.54s/it, loss=0.0633]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:20<00:29,  1.54s/it, loss=0.17]  \u001b[A\n",
            " 84%|████████▎ | 92/110 [02:20<00:27,  1.53s/it, loss=0.17]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:21<00:27,  1.53s/it, loss=0.0683]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:21<00:25,  1.53s/it, loss=0.0683]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:23<00:25,  1.53s/it, loss=0.397] \u001b[A\n",
            " 85%|████████▌ | 94/110 [02:23<00:24,  1.53s/it, loss=0.397]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:24<00:24,  1.53s/it, loss=0.18] \u001b[A\n",
            " 86%|████████▋ | 95/110 [02:24<00:22,  1.52s/it, loss=0.18]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:26<00:22,  1.52s/it, loss=0.0576]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:26<00:21,  1.54s/it, loss=0.0576]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:27<00:21,  1.54s/it, loss=0.0787]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:27<00:19,  1.53s/it, loss=0.0787]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:29<00:19,  1.53s/it, loss=0.205] \u001b[A\n",
            " 89%|████████▉ | 98/110 [02:29<00:18,  1.53s/it, loss=0.205]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:31<00:18,  1.53s/it, loss=0.201]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:31<00:16,  1.53s/it, loss=0.201]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:32<00:16,  1.53s/it, loss=0.237]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:32<00:15,  1.53s/it, loss=0.237]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:34<00:15,  1.53s/it, loss=0.0887]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:34<00:13,  1.54s/it, loss=0.0887]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:35<00:13,  1.54s/it, loss=0.237] \u001b[A\n",
            " 93%|█████████▎| 102/110 [02:35<00:12,  1.53s/it, loss=0.237]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:37<00:12,  1.53s/it, loss=0.114]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:37<00:10,  1.53s/it, loss=0.114]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:38<00:10,  1.53s/it, loss=0.238]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:38<00:09,  1.52s/it, loss=0.238]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:40<00:09,  1.52s/it, loss=0.366]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:40<00:07,  1.52s/it, loss=0.366]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:41<00:07,  1.52s/it, loss=0.193]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:41<00:06,  1.54s/it, loss=0.193]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:43<00:06,  1.54s/it, loss=0.0996]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:43<00:04,  1.53s/it, loss=0.0996]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:44<00:04,  1.53s/it, loss=0.162] \u001b[A\n",
            " 98%|█████████▊| 108/110 [02:44<00:03,  1.53s/it, loss=0.162]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:46<00:03,  1.53s/it, loss=0.165]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.53s/it, loss=0.165]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.53s/it, loss=0.0126]\u001b[A\n",
            "100%|██████████| 110/110 [02:46<00:00,  1.22s/it, loss=0.0126]\u001b[A\n",
            "                                                              \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:13,  1.94it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:03<00:11,  1.90it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:11,  1.90it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:04<00:10,  1.90it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:10,  1.90it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:05<00:09,  1.90it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  1.90it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:06<00:08,  1.90it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  1.90it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:07<00:07,  1.90it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  1.90it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:08<00:06,  1.90it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  1.90it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:09<00:05,  1.90it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:10<00:04,  1.90it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:10<00:04,  1.90it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:11<00:03,  1.90it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:11<00:03,  1.90it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:12<00:02,  1.90it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:12<00:02,  1.90it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:14<00:00,  1.90it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:14<00:00,  2.35it/s]\u001b[A\n",
            " 83%|████████▎ | 5/6 [13:12<02:33, 153.81s/it]\n",
            "  0%|          | 0/110 [00:00<?, ?it/s]\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.9391, F1 score @0.5 = 0.5025\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "  0%|          | 0/110 [00:01<?, ?it/s, loss=0.155]\u001b[A\n",
            "  1%|          | 1/110 [00:01<02:49,  1.55s/it, loss=0.155]\u001b[A\n",
            "  1%|          | 1/110 [00:03<02:49,  1.55s/it, loss=0.0673]\u001b[A\n",
            "  2%|▏         | 2/110 [00:03<02:46,  1.54s/it, loss=0.0673]\u001b[A\n",
            "  2%|▏         | 2/110 [00:04<02:46,  1.54s/it, loss=0.0212]\u001b[A\n",
            "  3%|▎         | 3/110 [00:04<02:44,  1.53s/it, loss=0.0212]\u001b[A\n",
            "  3%|▎         | 3/110 [00:06<02:44,  1.53s/it, loss=0.155] \u001b[A\n",
            "  4%|▎         | 4/110 [00:06<02:42,  1.53s/it, loss=0.155]\u001b[A\n",
            "  4%|▎         | 4/110 [00:07<02:42,  1.53s/it, loss=0.157]\u001b[A\n",
            "  5%|▍         | 5/110 [00:07<02:39,  1.52s/it, loss=0.157]\u001b[A\n",
            "  5%|▍         | 5/110 [00:09<02:39,  1.52s/it, loss=0.0589]\u001b[A\n",
            "  5%|▌         | 6/110 [00:09<02:39,  1.53s/it, loss=0.0589]\u001b[A\n",
            "  5%|▌         | 6/110 [00:10<02:39,  1.53s/it, loss=0.203] \u001b[A\n",
            "  6%|▋         | 7/110 [00:10<02:37,  1.53s/it, loss=0.203]\u001b[A\n",
            "  6%|▋         | 7/110 [00:12<02:37,  1.53s/it, loss=0.285]\u001b[A\n",
            "  7%|▋         | 8/110 [00:12<02:35,  1.53s/it, loss=0.285]\u001b[A\n",
            "  7%|▋         | 8/110 [00:13<02:35,  1.53s/it, loss=0.179]\u001b[A\n",
            "  8%|▊         | 9/110 [00:13<02:33,  1.52s/it, loss=0.179]\u001b[A\n",
            "  8%|▊         | 9/110 [00:15<02:33,  1.52s/it, loss=0.11] \u001b[A\n",
            "  9%|▉         | 10/110 [00:15<02:31,  1.52s/it, loss=0.11]\u001b[A\n",
            "  9%|▉         | 10/110 [00:16<02:31,  1.52s/it, loss=0.0293]\u001b[A\n",
            " 10%|█         | 11/110 [00:16<02:31,  1.53s/it, loss=0.0293]\u001b[A\n",
            " 10%|█         | 11/110 [00:18<02:31,  1.53s/it, loss=0.0711]\u001b[A\n",
            " 11%|█         | 12/110 [00:18<02:29,  1.52s/it, loss=0.0711]\u001b[A\n",
            " 11%|█         | 12/110 [00:19<02:29,  1.52s/it, loss=0.201] \u001b[A\n",
            " 12%|█▏        | 13/110 [00:19<02:27,  1.52s/it, loss=0.201]\u001b[A\n",
            " 12%|█▏        | 13/110 [00:21<02:27,  1.52s/it, loss=0.26] \u001b[A\n",
            " 13%|█▎        | 14/110 [00:21<02:25,  1.52s/it, loss=0.26]\u001b[A\n",
            " 13%|█▎        | 14/110 [00:22<02:25,  1.52s/it, loss=0.183]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:22<02:23,  1.52s/it, loss=0.183]\u001b[A\n",
            " 14%|█▎        | 15/110 [00:24<02:23,  1.52s/it, loss=0.138]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:24<02:23,  1.53s/it, loss=0.138]\u001b[A\n",
            " 15%|█▍        | 16/110 [00:25<02:23,  1.53s/it, loss=0.206]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:25<02:21,  1.52s/it, loss=0.206]\u001b[A\n",
            " 15%|█▌        | 17/110 [00:27<02:21,  1.52s/it, loss=0.198]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:27<02:19,  1.52s/it, loss=0.198]\u001b[A\n",
            " 16%|█▋        | 18/110 [00:28<02:19,  1.52s/it, loss=0.153]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:28<02:17,  1.51s/it, loss=0.153]\u001b[A\n",
            " 17%|█▋        | 19/110 [00:30<02:17,  1.51s/it, loss=0.072]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:30<02:16,  1.51s/it, loss=0.072]\u001b[A\n",
            " 18%|█▊        | 20/110 [00:31<02:16,  1.51s/it, loss=0.145]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:31<02:15,  1.53s/it, loss=0.145]\u001b[A\n",
            " 19%|█▉        | 21/110 [00:33<02:15,  1.53s/it, loss=0.17] \u001b[A\n",
            " 20%|██        | 22/110 [00:33<02:13,  1.52s/it, loss=0.17]\u001b[A\n",
            " 20%|██        | 22/110 [00:34<02:13,  1.52s/it, loss=0.353]\u001b[A\n",
            " 21%|██        | 23/110 [00:34<02:12,  1.52s/it, loss=0.353]\u001b[A\n",
            " 21%|██        | 23/110 [00:36<02:12,  1.52s/it, loss=0.0167]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:36<02:10,  1.52s/it, loss=0.0167]\u001b[A\n",
            " 22%|██▏       | 24/110 [00:38<02:10,  1.52s/it, loss=0.0519]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:38<02:08,  1.52s/it, loss=0.0519]\u001b[A\n",
            " 23%|██▎       | 25/110 [00:39<02:08,  1.52s/it, loss=0.321] \u001b[A\n",
            " 24%|██▎       | 26/110 [00:39<02:08,  1.53s/it, loss=0.321]\u001b[A\n",
            " 24%|██▎       | 26/110 [00:41<02:08,  1.53s/it, loss=0.227]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:41<02:05,  1.52s/it, loss=0.227]\u001b[A\n",
            " 25%|██▍       | 27/110 [00:42<02:05,  1.52s/it, loss=0.197]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:42<02:04,  1.52s/it, loss=0.197]\u001b[A\n",
            " 25%|██▌       | 28/110 [00:44<02:04,  1.52s/it, loss=0.129]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:44<02:02,  1.51s/it, loss=0.129]\u001b[A\n",
            " 26%|██▋       | 29/110 [00:45<02:02,  1.51s/it, loss=0.0921]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:45<02:01,  1.51s/it, loss=0.0921]\u001b[A\n",
            " 27%|██▋       | 30/110 [00:47<02:01,  1.51s/it, loss=0.106] \u001b[A\n",
            " 28%|██▊       | 31/110 [00:47<02:00,  1.53s/it, loss=0.106]\u001b[A\n",
            " 28%|██▊       | 31/110 [00:48<02:00,  1.53s/it, loss=0.346]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:48<01:58,  1.52s/it, loss=0.346]\u001b[A\n",
            " 29%|██▉       | 32/110 [00:50<01:58,  1.52s/it, loss=0.0935]\u001b[A\n",
            " 30%|███       | 33/110 [00:50<01:56,  1.52s/it, loss=0.0935]\u001b[A\n",
            " 30%|███       | 33/110 [00:51<01:56,  1.52s/it, loss=0.184] \u001b[A\n",
            " 31%|███       | 34/110 [00:51<01:54,  1.51s/it, loss=0.184]\u001b[A\n",
            " 31%|███       | 34/110 [00:53<01:54,  1.51s/it, loss=0.0667]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:53<01:53,  1.51s/it, loss=0.0667]\u001b[A\n",
            " 32%|███▏      | 35/110 [00:54<01:53,  1.51s/it, loss=0.0377]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:54<01:52,  1.52s/it, loss=0.0377]\u001b[A\n",
            " 33%|███▎      | 36/110 [00:56<01:52,  1.52s/it, loss=0.114] \u001b[A\n",
            " 34%|███▎      | 37/110 [00:56<01:50,  1.52s/it, loss=0.114]\u001b[A\n",
            " 34%|███▎      | 37/110 [00:57<01:50,  1.52s/it, loss=0.0447]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:57<01:49,  1.52s/it, loss=0.0447]\u001b[A\n",
            " 35%|███▍      | 38/110 [00:59<01:49,  1.52s/it, loss=0.0777]\u001b[A\n",
            " 35%|███▌      | 39/110 [00:59<01:47,  1.51s/it, loss=0.0777]\u001b[A\n",
            " 35%|███▌      | 39/110 [01:00<01:47,  1.51s/it, loss=0.207] \u001b[A\n",
            " 36%|███▋      | 40/110 [01:00<01:45,  1.51s/it, loss=0.207]\u001b[A\n",
            " 36%|███▋      | 40/110 [01:02<01:45,  1.51s/it, loss=0.194]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:02<01:45,  1.52s/it, loss=0.194]\u001b[A\n",
            " 37%|███▋      | 41/110 [01:03<01:45,  1.52s/it, loss=0.114]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:03<01:43,  1.52s/it, loss=0.114]\u001b[A\n",
            " 38%|███▊      | 42/110 [01:05<01:43,  1.52s/it, loss=0.163]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:05<01:41,  1.52s/it, loss=0.163]\u001b[A\n",
            " 39%|███▉      | 43/110 [01:06<01:41,  1.52s/it, loss=0.323]\u001b[A\n",
            " 40%|████      | 44/110 [01:06<01:39,  1.51s/it, loss=0.323]\u001b[A\n",
            " 40%|████      | 44/110 [01:08<01:39,  1.51s/it, loss=0.1]  \u001b[A\n",
            " 41%|████      | 45/110 [01:08<01:38,  1.51s/it, loss=0.1]\u001b[A\n",
            " 41%|████      | 45/110 [01:09<01:38,  1.51s/it, loss=0.136]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:09<01:37,  1.53s/it, loss=0.136]\u001b[A\n",
            " 42%|████▏     | 46/110 [01:11<01:37,  1.53s/it, loss=0.191]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:11<01:35,  1.52s/it, loss=0.191]\u001b[A\n",
            " 43%|████▎     | 47/110 [01:12<01:35,  1.52s/it, loss=0.158]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:12<01:33,  1.52s/it, loss=0.158]\u001b[A\n",
            " 44%|████▎     | 48/110 [01:14<01:33,  1.52s/it, loss=0.0893]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:14<01:32,  1.51s/it, loss=0.0893]\u001b[A\n",
            " 45%|████▍     | 49/110 [01:15<01:32,  1.51s/it, loss=0.0906]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:15<01:30,  1.51s/it, loss=0.0906]\u001b[A\n",
            " 45%|████▌     | 50/110 [01:17<01:30,  1.51s/it, loss=0.057] \u001b[A\n",
            " 46%|████▋     | 51/110 [01:17<01:29,  1.52s/it, loss=0.057]\u001b[A\n",
            " 46%|████▋     | 51/110 [01:18<01:29,  1.52s/it, loss=0.0665]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:19<01:28,  1.52s/it, loss=0.0665]\u001b[A\n",
            " 47%|████▋     | 52/110 [01:20<01:28,  1.52s/it, loss=0.2]   \u001b[A\n",
            " 48%|████▊     | 53/110 [01:20<01:26,  1.52s/it, loss=0.2]\u001b[A\n",
            " 48%|████▊     | 53/110 [01:22<01:26,  1.52s/it, loss=0.226]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:22<01:24,  1.52s/it, loss=0.226]\u001b[A\n",
            " 49%|████▉     | 54/110 [01:23<01:24,  1.52s/it, loss=0.465]\u001b[A\n",
            " 50%|█████     | 55/110 [01:23<01:23,  1.52s/it, loss=0.465]\u001b[A\n",
            " 50%|█████     | 55/110 [01:25<01:23,  1.52s/it, loss=0.0561]\u001b[A\n",
            " 51%|█████     | 56/110 [01:25<01:22,  1.53s/it, loss=0.0561]\u001b[A\n",
            " 51%|█████     | 56/110 [01:26<01:22,  1.53s/it, loss=0.063] \u001b[A\n",
            " 52%|█████▏    | 57/110 [01:26<01:20,  1.52s/it, loss=0.063]\u001b[A\n",
            " 52%|█████▏    | 57/110 [01:28<01:20,  1.52s/it, loss=0.0355]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:28<01:18,  1.52s/it, loss=0.0355]\u001b[A\n",
            " 53%|█████▎    | 58/110 [01:29<01:18,  1.52s/it, loss=0.129] \u001b[A\n",
            " 54%|█████▎    | 59/110 [01:29<01:17,  1.51s/it, loss=0.129]\u001b[A\n",
            " 54%|█████▎    | 59/110 [01:31<01:17,  1.51s/it, loss=0.0945]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:31<01:15,  1.51s/it, loss=0.0945]\u001b[A\n",
            " 55%|█████▍    | 60/110 [01:32<01:15,  1.51s/it, loss=0.188] \u001b[A\n",
            " 55%|█████▌    | 61/110 [01:32<01:14,  1.53s/it, loss=0.188]\u001b[A\n",
            " 55%|█████▌    | 61/110 [01:34<01:14,  1.53s/it, loss=0.127]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:34<01:12,  1.52s/it, loss=0.127]\u001b[A\n",
            " 56%|█████▋    | 62/110 [01:35<01:12,  1.52s/it, loss=0.101]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:35<01:11,  1.52s/it, loss=0.101]\u001b[A\n",
            " 57%|█████▋    | 63/110 [01:37<01:11,  1.52s/it, loss=0.0584]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:37<01:09,  1.51s/it, loss=0.0584]\u001b[A\n",
            " 58%|█████▊    | 64/110 [01:38<01:09,  1.51s/it, loss=0.113] \u001b[A\n",
            " 59%|█████▉    | 65/110 [01:38<01:08,  1.51s/it, loss=0.113]\u001b[A\n",
            " 59%|█████▉    | 65/110 [01:40<01:08,  1.51s/it, loss=0.151]\u001b[A\n",
            " 60%|██████    | 66/110 [01:40<01:07,  1.53s/it, loss=0.151]\u001b[A\n",
            " 60%|██████    | 66/110 [01:41<01:07,  1.53s/it, loss=0.12] \u001b[A\n",
            " 61%|██████    | 67/110 [01:41<01:05,  1.52s/it, loss=0.12]\u001b[A\n",
            " 61%|██████    | 67/110 [01:43<01:05,  1.52s/it, loss=0.0648]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:43<01:03,  1.52s/it, loss=0.0648]\u001b[A\n",
            " 62%|██████▏   | 68/110 [01:44<01:03,  1.52s/it, loss=0.0726]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:44<01:02,  1.52s/it, loss=0.0726]\u001b[A\n",
            " 63%|██████▎   | 69/110 [01:46<01:02,  1.52s/it, loss=0.075] \u001b[A\n",
            " 64%|██████▎   | 70/110 [01:46<01:00,  1.52s/it, loss=0.075]\u001b[A\n",
            " 64%|██████▎   | 70/110 [01:47<01:00,  1.52s/it, loss=0.101]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:47<00:59,  1.53s/it, loss=0.101]\u001b[A\n",
            " 65%|██████▍   | 71/110 [01:49<00:59,  1.53s/it, loss=0.274]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:49<00:57,  1.52s/it, loss=0.274]\u001b[A\n",
            " 65%|██████▌   | 72/110 [01:50<00:57,  1.52s/it, loss=0.115]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:50<00:56,  1.52s/it, loss=0.115]\u001b[A\n",
            " 66%|██████▋   | 73/110 [01:52<00:56,  1.52s/it, loss=0.0831]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:52<00:54,  1.52s/it, loss=0.0831]\u001b[A\n",
            " 67%|██████▋   | 74/110 [01:53<00:54,  1.52s/it, loss=0.14]  \u001b[A\n",
            " 68%|██████▊   | 75/110 [01:53<00:53,  1.52s/it, loss=0.14]\u001b[A\n",
            " 68%|██████▊   | 75/110 [01:55<00:53,  1.52s/it, loss=0.122]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:55<00:51,  1.53s/it, loss=0.122]\u001b[A\n",
            " 69%|██████▉   | 76/110 [01:56<00:51,  1.53s/it, loss=0.0719]\u001b[A\n",
            " 70%|███████   | 77/110 [01:56<00:50,  1.52s/it, loss=0.0719]\u001b[A\n",
            " 70%|███████   | 77/110 [01:58<00:50,  1.52s/it, loss=0.0871]\u001b[A\n",
            " 71%|███████   | 78/110 [01:58<00:48,  1.52s/it, loss=0.0871]\u001b[A\n",
            " 71%|███████   | 78/110 [02:00<00:48,  1.52s/it, loss=0.151] \u001b[A\n",
            " 72%|███████▏  | 79/110 [02:00<00:46,  1.52s/it, loss=0.151]\u001b[A\n",
            " 72%|███████▏  | 79/110 [02:01<00:46,  1.52s/it, loss=0.197]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:01<00:45,  1.52s/it, loss=0.197]\u001b[A\n",
            " 73%|███████▎  | 80/110 [02:03<00:45,  1.52s/it, loss=0.131]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:03<00:44,  1.53s/it, loss=0.131]\u001b[A\n",
            " 74%|███████▎  | 81/110 [02:04<00:44,  1.53s/it, loss=0.041]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:04<00:42,  1.52s/it, loss=0.041]\u001b[A\n",
            " 75%|███████▍  | 82/110 [02:06<00:42,  1.52s/it, loss=0.198]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:06<00:41,  1.52s/it, loss=0.198]\u001b[A\n",
            " 75%|███████▌  | 83/110 [02:07<00:41,  1.52s/it, loss=0.142]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:07<00:39,  1.52s/it, loss=0.142]\u001b[A\n",
            " 76%|███████▋  | 84/110 [02:09<00:39,  1.52s/it, loss=0.164]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:09<00:37,  1.52s/it, loss=0.164]\u001b[A\n",
            " 77%|███████▋  | 85/110 [02:10<00:37,  1.52s/it, loss=0.16] \u001b[A\n",
            " 78%|███████▊  | 86/110 [02:10<00:36,  1.53s/it, loss=0.16]\u001b[A\n",
            " 78%|███████▊  | 86/110 [02:12<00:36,  1.53s/it, loss=0.312]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:12<00:35,  1.52s/it, loss=0.312]\u001b[A\n",
            " 79%|███████▉  | 87/110 [02:13<00:35,  1.52s/it, loss=0.0657]\u001b[A\n",
            " 80%|████████  | 88/110 [02:13<00:33,  1.52s/it, loss=0.0657]\u001b[A\n",
            " 80%|████████  | 88/110 [02:15<00:33,  1.52s/it, loss=0.15]  \u001b[A\n",
            " 81%|████████  | 89/110 [02:15<00:31,  1.52s/it, loss=0.15]\u001b[A\n",
            " 81%|████████  | 89/110 [02:16<00:31,  1.52s/it, loss=0.0225]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:16<00:30,  1.51s/it, loss=0.0225]\u001b[A\n",
            " 82%|████████▏ | 90/110 [02:18<00:30,  1.51s/it, loss=0.0872]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:18<00:29,  1.53s/it, loss=0.0872]\u001b[A\n",
            " 83%|████████▎ | 91/110 [02:19<00:29,  1.53s/it, loss=0.152] \u001b[A\n",
            " 84%|████████▎ | 92/110 [02:19<00:27,  1.52s/it, loss=0.152]\u001b[A\n",
            " 84%|████████▎ | 92/110 [02:21<00:27,  1.52s/it, loss=0.0861]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:21<00:25,  1.52s/it, loss=0.0861]\u001b[A\n",
            " 85%|████████▍ | 93/110 [02:22<00:25,  1.52s/it, loss=0.135] \u001b[A\n",
            " 85%|████████▌ | 94/110 [02:22<00:24,  1.52s/it, loss=0.135]\u001b[A\n",
            " 85%|████████▌ | 94/110 [02:24<00:24,  1.52s/it, loss=0.0336]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:24<00:22,  1.51s/it, loss=0.0336]\u001b[A\n",
            " 86%|████████▋ | 95/110 [02:25<00:22,  1.51s/it, loss=0.216] \u001b[A\n",
            " 87%|████████▋ | 96/110 [02:25<00:21,  1.53s/it, loss=0.216]\u001b[A\n",
            " 87%|████████▋ | 96/110 [02:27<00:21,  1.53s/it, loss=0.0311]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:27<00:19,  1.52s/it, loss=0.0311]\u001b[A\n",
            " 88%|████████▊ | 97/110 [02:28<00:19,  1.52s/it, loss=0.101] \u001b[A\n",
            " 89%|████████▉ | 98/110 [02:28<00:18,  1.52s/it, loss=0.101]\u001b[A\n",
            " 89%|████████▉ | 98/110 [02:30<00:18,  1.52s/it, loss=0.0155]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:30<00:16,  1.52s/it, loss=0.0155]\u001b[A\n",
            " 90%|█████████ | 99/110 [02:31<00:16,  1.52s/it, loss=0.218] \u001b[A\n",
            " 91%|█████████ | 100/110 [02:31<00:15,  1.52s/it, loss=0.218]\u001b[A\n",
            " 91%|█████████ | 100/110 [02:33<00:15,  1.52s/it, loss=0.181]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:33<00:13,  1.53s/it, loss=0.181]\u001b[A\n",
            " 92%|█████████▏| 101/110 [02:35<00:13,  1.53s/it, loss=0.0433]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:35<00:12,  1.52s/it, loss=0.0433]\u001b[A\n",
            " 93%|█████████▎| 102/110 [02:36<00:12,  1.52s/it, loss=0.239] \u001b[A\n",
            " 94%|█████████▎| 103/110 [02:36<00:10,  1.52s/it, loss=0.239]\u001b[A\n",
            " 94%|█████████▎| 103/110 [02:38<00:10,  1.52s/it, loss=0.169]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:38<00:09,  1.52s/it, loss=0.169]\u001b[A\n",
            " 95%|█████████▍| 104/110 [02:39<00:09,  1.52s/it, loss=0.0689]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:39<00:07,  1.52s/it, loss=0.0689]\u001b[A\n",
            " 95%|█████████▌| 105/110 [02:41<00:07,  1.52s/it, loss=0.0844]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:41<00:06,  1.53s/it, loss=0.0844]\u001b[A\n",
            " 96%|█████████▋| 106/110 [02:42<00:06,  1.53s/it, loss=0.0326]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:42<00:04,  1.52s/it, loss=0.0326]\u001b[A\n",
            " 97%|█████████▋| 107/110 [02:44<00:04,  1.52s/it, loss=0.26]  \u001b[A\n",
            " 98%|█████████▊| 108/110 [02:44<00:03,  1.52s/it, loss=0.26]\u001b[A\n",
            " 98%|█████████▊| 108/110 [02:45<00:03,  1.52s/it, loss=0.204]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:45<00:01,  1.52s/it, loss=0.204]\u001b[A\n",
            " 99%|█████████▉| 109/110 [02:46<00:01,  1.52s/it, loss=0.0488]\u001b[A\n",
            "100%|██████████| 110/110 [02:46<00:00,  1.21s/it, loss=0.0488]\u001b[A\n",
            "                                                              \u001b[A\n",
            "  0%|          | 0/28 [00:00<?, ?it/s]\u001b[A\n",
            "  4%|▎         | 1/28 [00:00<00:13,  1.93it/s]\u001b[A\n",
            "  7%|▋         | 2/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 11%|█         | 3/28 [00:01<00:13,  1.92it/s]\u001b[A\n",
            " 14%|█▍        | 4/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 18%|█▊        | 5/28 [00:02<00:12,  1.91it/s]\u001b[A\n",
            " 21%|██▏       | 6/28 [00:03<00:11,  1.91it/s]\u001b[A\n",
            " 25%|██▌       | 7/28 [00:03<00:11,  1.91it/s]\u001b[A\n",
            " 29%|██▊       | 8/28 [00:04<00:10,  1.91it/s]\u001b[A\n",
            " 32%|███▏      | 9/28 [00:04<00:09,  1.91it/s]\u001b[A\n",
            " 36%|███▌      | 10/28 [00:05<00:09,  1.91it/s]\u001b[A\n",
            " 39%|███▉      | 11/28 [00:05<00:08,  1.90it/s]\u001b[A\n",
            " 43%|████▎     | 12/28 [00:06<00:08,  1.90it/s]\u001b[A\n",
            " 46%|████▋     | 13/28 [00:06<00:07,  1.90it/s]\u001b[A\n",
            " 50%|█████     | 14/28 [00:07<00:07,  1.90it/s]\u001b[A\n",
            " 54%|█████▎    | 15/28 [00:07<00:06,  1.91it/s]\u001b[A\n",
            " 57%|█████▋    | 16/28 [00:08<00:06,  1.91it/s]\u001b[A\n",
            " 61%|██████    | 17/28 [00:08<00:05,  1.91it/s]\u001b[A\n",
            " 64%|██████▍   | 18/28 [00:09<00:05,  1.91it/s]\u001b[A\n",
            " 68%|██████▊   | 19/28 [00:09<00:04,  1.91it/s]\u001b[A\n",
            " 71%|███████▏  | 20/28 [00:10<00:04,  1.91it/s]\u001b[A\n",
            " 75%|███████▌  | 21/28 [00:11<00:03,  1.91it/s]\u001b[A\n",
            " 79%|███████▊  | 22/28 [00:11<00:03,  1.91it/s]\u001b[A\n",
            " 82%|████████▏ | 23/28 [00:12<00:02,  1.91it/s]\u001b[A\n",
            " 86%|████████▌ | 24/28 [00:12<00:02,  1.91it/s]\u001b[A\n",
            " 89%|████████▉ | 25/28 [00:13<00:01,  1.90it/s]\u001b[A\n",
            " 93%|█████████▎| 26/28 [00:13<00:01,  1.91it/s]\u001b[A\n",
            " 96%|█████████▋| 27/28 [00:14<00:00,  1.90it/s]\u001b[A\n",
            "100%|██████████| 28/28 [00:14<00:00,  2.35it/s]\u001b[A\n",
            "100%|██████████| 6/6 [16:12<00:00, 162.11s/it]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "AUC = 0.9439, F1 score @0.5 = 0.5946\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Khh653ZJA-H"
      },
      "source": [
        "torch.save(model_bert.state_dict(),os.path.join(path, f'phobert_model.bin'))"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WmJzznjlXVk5",
        "outputId": "cdd354d4-acd9-40e9-81cc-c6ce82964d8b"
      },
      "source": [
        "test_df = test[['id','post_message']].fillna('none')\n",
        "test_df.post_message = test_df.post_message.progress_apply(lambda x: ' '.join([' '.join(sent) for sent in rdrsegmenter.tokenize(x)])) \n",
        "X_test = convert_lines(test_df, vocab, bpe,kmax_sequence_length)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1646/1646 [00:11<00:00, 148.59it/s]\n",
            "100%|██████████| 1646/1646 [00:02<00:00, 595.10it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-RbcaHecj2X",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c113e54f-03ee-49dc-95fb-b33d933980a1"
      },
      "source": [
        "preds_en = []\n",
        "for fold in range(5):\n",
        "    print(f\"Predicting for fold {fold}\")\n",
        "    preds_fold = []\n",
        "    model_bert.load_state_dict(torch.load(os.path.join(path, f'phobert_model.bin')))\n",
        "    test_dataset = torch.utils.data.TensorDataset(torch.tensor(X_test,dtype=torch.long))\n",
        "    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=kbatch_size, shuffle=False)\n",
        "    model_bert.eval()\n",
        "    pbar = tqdm(enumerate(test_loader),total=len(test_loader),leave=False)\n",
        "    for i, (x_batch,) in pbar:\n",
        "        y_pred = model_bert(x_batch.cuda(), attention_mask=(x_batch>0).cuda())\n",
        "        y_pred = y_pred.view(-1).detach().cpu().numpy()\n",
        "        preds_fold = np.concatenate([preds_fold, y_pred])\n",
        "    preds_fold = sigmoid(preds_fold)\n",
        "    preds_en.append(preds_fold)\n",
        "preds_en = np.mean(preds_en,axis=0)\n",
        "#test_df[\"label\"] = (preds_en > 0.5).astype(np.int)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predicting for fold 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicting for fold 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicting for fold 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicting for fold 3\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Predicting for fold 4\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            ""
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-S0e7a5GMdt3"
      },
      "source": [
        "x = pd.DataFrame({'test_id':test_df['id'],'label probability':preds_en})"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jcZMMCnAOIEW"
      },
      "source": [
        "x.to_csv('results.csv',index=False,header=False)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aN9R3F8VDv3Y"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}