LLM finetuning classification -fine tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8275a144",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import datasets\n",
    "import transformers\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "device = torch.device(\"mps\")\n",
    "from sklearn.metrics import f1_score,accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ff78e9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('new_train.csv',index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3f81c008",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;prompt&gt;[\"Is it morally right to try to have a...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;prompt&gt;[\"What is the difference between marri...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;prompt&gt;[\"explain function calling. how would ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>&lt;prompt&gt;[\"How can I create a test set for a ve...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>&lt;prompt&gt;[\"What is the best way to travel from ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  label\n",
       "0  <prompt>[\"Is it morally right to try to have a...      1\n",
       "1  <prompt>[\"What is the difference between marri...      2\n",
       "2  <prompt>[\"explain function calling. how would ...      0\n",
       "3  <prompt>[\"How can I create a test set for a ve...      1\n",
       "4  <prompt>[\"What is the best way to travel from ...      2"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e0e6b365",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['wrd_len'] = df['text'].str.split().apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b36e9bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "590.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['wrd_len'].quantile(0.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e5e82866",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label', ylabel='count'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAGwCAYAAAC0HlECAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAx+klEQVR4nO3df1iUdb7/8deAzaAFGCkgR1TK1p+APzKaSsNiGY3LXTaPW2plRZodrJSOctjLCLVdXF1TS8vjltGe1aPZSSv1UkdKyBwrSTQs2TRc2ksH+yWTVIAw3z/O4f42iz/uCJ1Bno/ruq+L+/N5zz3vD3Hl67rvD4PF6/V6BQAAgHMK8ncDAAAAbQGhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJjQwd8NXCoaGxt17NgxhYaGymKx+LsdAABggtfr1bfffquYmBgFBZ37XhKhqZUcO3ZMsbGx/m4DAAC0wOeff67u3bufs4bQ1EpCQ0Ml/e83PSwszM/dAAAAMzwej2JjY41/x8+F0NRKmh7JhYWFEZoAAGhjzGytYSM4AACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEzwa2jKz8/XsGHDFBoaqsjISKWnp6u8vNyn5ocfflBmZqauuuoqXXHFFRo7dqyqqqp8aiorK5WWlqZOnTopMjJSM2fO1OnTp31qdu7cqSFDhshms6l3794qKCho1s/y5cvVq1cvhYSEKCkpSe+//36rrxkAALRNfg1NRUVFyszM1J49e+R0OlVfX6/U1FTV1NQYNTNmzNCbb76p9evXq6ioSMeOHdMdd9xhzDc0NCgtLU11dXXavXu3Xn75ZRUUFCg3N9eoqaioUFpamkaOHKnS0lJNnz5dDz74oLZt22bUrFu3TllZWXryySf14YcfKjExUQ6HQydOnLg43wwAABDYvAHkxIkTXkneoqIir9fr9Z48edJ72WWXedevX2/UfPLJJ15JXpfL5fV6vd4tW7Z4g4KCvG6326h5/vnnvWFhYd7a2lqv1+v1zpo1yztgwACf97rzzju9DofDOL/++uu9mZmZxnlDQ4M3JibGm5+fb6r36upqryRvdXX1T1w1AADwl5/y73cH/0Y2X9XV1ZKkiIgISVJJSYnq6+uVkpJi1PTt21c9evSQy+XSDTfcIJfLpfj4eEVFRRk1DodDDz/8sA4ePKjBgwfL5XL5XKOpZvr06ZKkuro6lZSUKCcnx5gPCgpSSkqKXC7XGXutra1VbW2tce7xeH7e4oF2btBTef5uoU0rnZ3n7xaAS17AbARvbGzU9OnTddNNN2ngwIGSJLfbLavVqs6dO/vURkVFye12GzU/DkxN801z56rxeDz6/vvv9eWXX6qhoeGMNU3X+Gf5+fkKDw83jtjY2JYtHAAAtAkBE5oyMzNVVlamtWvX+rsVU3JyclRdXW0cn3/+ub9bAgAAF1BAPJ6bNm2aNm3apOLiYnXv3t0Yj46OVl1dnU6ePOlzt6mqqkrR0dFGzT//llvTb9f9uOaff+OuqqpKYWFh6tixo4KDgxUcHHzGmqZr/DObzSabzdayBQMAgDbHr3eavF6vpk2bpg0bNuitt95SXFycz/zQoUN12WWXqbCw0BgrLy9XZWWl7Ha7JMlut+ujjz7y+S03p9OpsLAw9e/f36j58TWaapquYbVaNXToUJ+axsZGFRYWGjUAAKB98+udpszMTK1Zs0avv/66QkNDjf1D4eHh6tixo8LDw5WRkaGsrCxFREQoLCxMjzzyiOx2u2644QZJUmpqqvr376977rlHCxYskNvt1uzZs5WZmWncCZo6daqWLVumWbNm6YEHHtBbb72lV155RZs3bzZ6ycrK0qRJk3Tdddfp+uuv15IlS1RTU6P777//4n9jAABAwPFraHr++eclScnJyT7jL730ku677z5J0uLFixUUFKSxY8eqtrZWDodDzz33nFEbHBysTZs26eGHH5bdbtfll1+uSZMmae7cuUZNXFycNm/erBkzZmjp0qXq3r27XnjhBTkcDqPmzjvv1BdffKHc3Fy53W4NGjRIW7dubbY5HAAAtE8Wr9fr9XcTlwKPx6Pw8HBVV1crLCzM3+0AbQ4fOfDz8JEDF8YjhY/5u4U269nblvq7BVN+yr/fAfPbcwAAAIGM0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACY0MHfDcA/Uu+c6+8W2qzt63L93QIAwA/8eqepuLhYY8aMUUxMjCwWizZu3Ogzb7FYzngsXLjQqOnVq1ez+fnz5/tc58CBAxo+fLhCQkIUGxurBQsWNOtl/fr16tu3r0JCQhQfH68tW7ZckDUDAIC2ya+hqaamRomJiVq+fPkZ548fP+5zrFq1ShaLRWPHjvWpmzt3rk/dI488Ysx5PB6lpqaqZ8+eKikp0cKFC5WXl6eVK1caNbt379b48eOVkZGhffv2KT09Xenp6SorK7swCwcAAG2OXx/PjR49WqNHjz7rfHR0tM/566+/rpEjR+rqq6/2GQ8NDW1W22T16tWqq6vTqlWrZLVaNWDAAJWWlurpp5/WlClTJElLly7VqFGjNHPmTEnSvHnz5HQ6tWzZMq1YseLnLBEAAFwi2sxG8KqqKm3evFkZGRnN5ubPn6+rrrpKgwcP1sKFC3X69GljzuVyacSIEbJarcaYw+FQeXm5vvnmG6MmJSXF55oOh0Mul+us/dTW1srj8fgcAADg0tVmNoK//PLLCg0N1R133OEz/uijj2rIkCGKiIjQ7t27lZOTo+PHj+vpp5+WJLndbsXFxfm8Jioqypi78sor5Xa7jbEf17jd7rP2k5+frzlz5rTG0gAAQBvQZkLTqlWrNHHiRIWEhPiMZ2VlGV8nJCTIarXqoYceUn5+vmw22wXrJycnx+e9PR6PYmNjL9j7AQAA/2oToemdd95ReXm51q1bd97apKQknT59WkePHlWfPn0UHR2tqqoqn5qm86Z9UGerOds+KUmy2WwXNJQBAIDA0ib2NL344osaOnSoEhMTz1tbWlqqoKAgRUZGSpLsdruKi4tVX19v1DidTvXp00dXXnmlUVNYWOhzHafTKbvd3oqrAAAAbZlfQ9OpU6dUWlqq0tJSSVJFRYVKS0tVWVlp1Hg8Hq1fv14PPvhgs9e7XC4tWbJE+/fv12effabVq1drxowZuvvuu41ANGHCBFmtVmVkZOjgwYNat26dli5d6vNo7bHHHtPWrVu1aNEiHTp0SHl5edq7d6+mTZt2Yb8BAACgzfDr47m9e/dq5MiRxnlTkJk0aZIKCgokSWvXrpXX69X48eObvd5ms2nt2rXKy8tTbW2t4uLiNGPGDJ9AFB4eru3btyszM1NDhw5Vly5dlJuba3zcgCTdeOONWrNmjWbPnq3f/e53uvbaa7Vx40YNHDjwAq0cAAC0NX4NTcnJyfJ6veesmTJlik/A+bEhQ4Zoz549532fhIQEvfPOO+esGTdunMaNG3feawEAgPapTexpAgAA8DdCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEv4am4uJijRkzRjExMbJYLNq4caPP/H333SeLxeJzjBo1yqfm66+/1sSJExUWFqbOnTsrIyNDp06d8qk5cOCAhg8frpCQEMXGxmrBggXNelm/fr369u2rkJAQxcfHa8uWLa2+XgAA0Hb5NTTV1NQoMTFRy5cvP2vNqFGjdPz4ceP47//+b5/5iRMn6uDBg3I6ndq0aZOKi4s1ZcoUY97j8Sg1NVU9e/ZUSUmJFi5cqLy8PK1cudKo2b17t8aPH6+MjAzt27dP6enpSk9PV1lZWesvGgAAtEkd/Pnmo0eP1ujRo89ZY7PZFB0dfca5Tz75RFu3btUHH3yg6667TpL07LPP6vbbb9ef/vQnxcTEaPXq1aqrq9OqVatktVo1YMAAlZaW6umnnzbC1dKlSzVq1CjNnDlTkjRv3jw5nU4tW7ZMK1asaMUVAwCAtirg9zTt3LlTkZGR6tOnjx5++GF99dVXxpzL5VLnzp2NwCRJKSkpCgoK0nvvvWfUjBgxQlar1ahxOBwqLy/XN998Y9SkpKT4vK/D4ZDL5TprX7W1tfJ4PD4HAAC4dAV0aBo1apT+8pe/qLCwUH/84x9VVFSk0aNHq6GhQZLkdrsVGRnp85oOHTooIiJCbrfbqImKivKpaTo/X03T/Jnk5+crPDzcOGJjY3/eYgEAQEDz6+O587nrrruMr+Pj45WQkKBrrrlGO3fu1G233ebHzqScnBxlZWUZ5x6Ph+AEAMAlLKDvNP2zq6++Wl26dNHhw4clSdHR0Tpx4oRPzenTp/X1118b+6Cio6NVVVXlU9N0fr6as+2lkv53r1VYWJjPAQAALl1tKjT94x//0FdffaVu3bpJkux2u06ePKmSkhKj5q233lJjY6OSkpKMmuLiYtXX1xs1TqdTffr00ZVXXmnUFBYW+ryX0+mU3W6/0EsCAABthF9D06lTp1RaWqrS0lJJUkVFhUpLS1VZWalTp05p5syZ2rNnj44eParCwkL9+te/Vu/eveVwOCRJ/fr106hRozR58mS9//77evfddzVt2jTdddddiomJkSRNmDBBVqtVGRkZOnjwoNatW6elS5f6PFp77LHHtHXrVi1atEiHDh1SXl6e9u7dq2nTpl307wkAAAhMfg1Ne/fu1eDBgzV48GBJUlZWlgYPHqzc3FwFBwfrwIED+tWvfqVf/OIXysjI0NChQ/XOO+/IZrMZ11i9erX69u2r2267Tbfffrtuvvlmn89gCg8P1/bt21VRUaGhQ4fq8ccfV25urs9nOd14441as2aNVq5cqcTERL366qvauHGjBg4cePG+GQAAIKD5dSN4cnKyvF7vWee3bdt23mtERERozZo156xJSEjQO++8c86acePGady4ced9PwAA0D61qT1NAAAA/kJoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJjg19BUXFysMWPGKCYmRhaLRRs3bjTm6uvrlZ2drfj4eF1++eWKiYnRvffeq2PHjvlco1evXrJYLD7H/PnzfWoOHDig4cOHKyQkRLGxsVqwYEGzXtavX6++ffsqJCRE8fHx2rJlywVZMwAAaJv8GppqamqUmJio5cuXN5v77rvv9OGHH+qJJ57Qhx9+qNdee03l5eX61a9+1ax27ty5On78uHE88sgjxpzH41Fqaqp69uypkpISLVy4UHl5eVq5cqVRs3v3bo0fP14ZGRnat2+f0tPTlZ6errKysguzcAAA0OZ08Oebjx49WqNHjz7jXHh4uJxOp8/YsmXLdP3116uyslI9evQwxkNDQxUdHX3G66xevVp1dXVatWqVrFarBgwYoNLSUj399NOaMmWKJGnp0qUaNWqUZs6cKUmaN2+enE6nli1bphUrVpzxurW1taqtrTXOPR6P+YUDAIA2p03taaqurpbFYlHnzp19xufPn6+rrrpKgwcP1sKFC3X69GljzuVyacSIEbJarcaYw+FQeXm5vvnmG6MmJSXF55oOh0Mul+usveTn5ys8PNw4YmNjW2GFAAAgULWZ0PTDDz8oOztb48ePV1hYmDH+6KOPau3atXr77bf10EMP6Q9/+INmzZplzLvdbkVFRflcq+nc7Xafs6Zp/kxycnJUXV1tHJ9//vnPXiMAAAhcfn08Z1Z9fb1++9vfyuv16vnnn/eZy8rKMr5OSEiQ1WrVQw89pPz8fNlstgvWk81mu6DXBwAAgSXg7zQ1Baa///3vcjqdPneZziQpKUmnT5/W0aNHJUnR0dGqqqryqWk6b9oHdbaas+2TAgAA7U9Ah6amwPTpp59qx44duuqqq877mtLSUgUFBSkyMlKSZLfbVVxcrPr6eqPG6XSqT58+uvLKK42awsJCn+s4nU7Z7fZWXA0AAGjL/Pp47tSpUzp8+LBxXlFRodLSUkVERKhbt27613/9V3344YfatGmTGhoajD1GERERslqtcrlceu+99zRy5EiFhobK5XJpxowZuvvuu41ANGHCBM2ZM0cZGRnKzs5WWVmZli5dqsWLFxvv+9hjj+mWW27RokWLlJaWprVr12rv3r0+H0sAAADaN7+Gpr1792rkyJHGedP+pEmTJikvL09vvPGGJGnQoEE+r3v77beVnJwsm82mtWvXKi8vT7W1tYqLi9OMGTN89jmFh4dr+/btyszM1NChQ9WlSxfl5uYaHzcgSTfeeKPWrFmj2bNn63e/+52uvfZabdy4UQMHDryAqwcAAG2JX0NTcnKyvF7vWefPNSdJQ4YM0Z49e877PgkJCXrnnXfOWTNu3DiNGzfuvNcCAADtU0DvaQIAAAgUhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAE1oUmm699VadPHmy2bjH49Gtt976c3sCAAAIOC0KTTt37lRdXV2z8R9++OG8f+MNAACgLfpJf7D3wIEDxtcff/yx3G63cd7Q0KCtW7fqX/7lX1qvOwAAgADxk0LToEGDZLFYZLFYzvgYrmPHjnr22WdbrTkAAIBA8ZNCU0VFhbxer66++mq9//776tq1qzFntVoVGRmp4ODgVm8SAADA335SaOrZs6ckqbGx8YI0AwAAEKh+Umj6sU8//VRvv/22Tpw40SxE5ebm/uzGAAAAAkmLQtOf//xnPfzww+rSpYuio6NlsViMOYvFQmgCAACXnBaFpqeeekq///3vlZ2d3dr9AAAABKQWfU7TN998o3HjxrV2LwAAAAGrRaFp3Lhx2r59e2v3AgAAELBa9Hiud+/eeuKJJ7Rnzx7Fx8frsssu85l/9NFHW6U5AACAQNGi0LRy5UpdccUVKioqUlFRkc+cxWIhNAEAgEtOi0JTRUVFa/cBAAAQ0Fq0pwkAAKC9adGdpgceeOCc86tWrWpRMwAAAIGqRaHpm2++8Tmvr69XWVmZTp48ecY/5AsAANDWtSg0bdiwodlYY2OjHn74YV1zzTU/uykAAIBA02p7moKCgpSVlaXFixe31iUBAAACRqtuBD9y5IhOnz7dmpcEAAAICC16PJeVleVz7vV6dfz4cW3evFmTJk1qlcYAAAACSYtC0759+3zOg4KC1LVrVy1atOi8v1kHAADQFrUoNL399tut3QcAAEBA+1l7mr744gvt2rVLu3bt0hdffPGTX19cXKwxY8YoJiZGFotFGzdu9Jn3er3Kzc1Vt27d1LFjR6WkpOjTTz/1qfn66681ceJEhYWFqXPnzsrIyNCpU6d8ag4cOKDhw4crJCREsbGxWrBgQbNe1q9fr759+yokJETx8fHasmXLT14PAAC4dLUoNNXU1OiBBx5Qt27dNGLECI0YMUIxMTHKyMjQd99995Ouk5iYqOXLl59xfsGCBXrmmWe0YsUKvffee7r88svlcDj0ww8/GDUTJ07UwYMH5XQ6tWnTJhUXF2vKlCnGvMfjUWpqqnr27KmSkhItXLhQeXl5WrlypVGze/dujR8/XhkZGdq3b5/S09OVnp6usrKyFnx3AADApahFoSkrK0tFRUV68803dfLkSZ08eVKvv/66ioqK9Pjjj5u+zujRo/XUU0/pN7/5TbM5r9erJUuWaPbs2fr1r3+thIQE/eUvf9GxY8eMO1KffPKJtm7dqhdeeEFJSUm6+eab9eyzz2rt2rU6duyYJGn16tWqq6vTqlWrNGDAAN1111169NFH9fTTTxvvtXTpUo0aNUozZ85Uv379NG/ePA0ZMkTLli07a++1tbXyeDw+BwAAuHS1KDT9z//8j1588UWNHj1aYWFhCgsL0+23364///nPevXVV1ulsYqKCrndbqWkpBhj4eHhSkpKksvlkiS5XC517txZ1113nVGTkpKioKAgvffee0bNiBEjZLVajRqHw6Hy8nLjk81dLpfP+zTVNL3PmeTn5ys8PNw4YmNjf/6iAQBAwGpRaPruu+8UFRXVbDwyMvInPZ47F7fbLUnN3icqKsqYc7vdioyM9Jnv0KGDIiIifGrOdI0fv8fZaprmzyQnJ0fV1dXG8fnnn//UJQIAgDakRaHJbrfrySef9Nlb9P3332vOnDmy2+2t1lwgs9lsxl22pgMAAFy6WvSRA0uWLNGoUaPUvXt3JSYmSpL2798vm82m7du3t0pj0dHRkqSqqip169bNGK+qqtKgQYOMmhMnTvi87vTp0/r666+N10dHR6uqqsqnpun8fDVN8wAAAC260xQfH69PP/1U+fn5GjRokAYNGqT58+fr8OHDGjBgQKs0FhcXp+joaBUWFhpjHo9H7733nnE3y2636+TJkyopKTFq3nrrLTU2NiopKcmoKS4uVn19vVHjdDrVp08fXXnllUbNj9+nqaa93DUDAADn16I7Tfn5+YqKitLkyZN9xletWqUvvvhC2dnZpq5z6tQpHT582DivqKhQaWmpIiIi1KNHD02fPl1PPfWUrr32WsXFxemJJ55QTEyM0tPTJUn9+vXTqFGjNHnyZK1YsUL19fWaNm2a7rrrLsXExEiSJkyYoDlz5igjI0PZ2dkqKyvT0qVLff6w8GOPPaZbbrlFixYtUlpamtauXau9e/f6fCwBAABo31p0p+k///M/1bdv32bjAwYM0IoVK0xfZ+/evRo8eLAGDx4s6X8/ymDw4MHKzc2VJM2aNUuPPPKIpkyZomHDhunUqVPaunWrQkJCjGusXr1affv21W233abbb79dN998s0/YCQ8P1/bt21VRUaGhQ4fq8ccfV25urs9nOd14441as2aNVq5cqcTERL366qvauHGjBg4c+JO/NwAA4NLUojtNbrfbZ59Rk65du+r48eOmr5OcnCyv13vWeYvForlz52ru3LlnrYmIiNCaNWvO+T4JCQl65513zlkzbtw4jRs37twNAwCAdqtFd5piY2P17rvvNht/9913jcdiAAAAl5IW3WmaPHmypk+frvr6et16662SpMLCQs2aNesnfSI4AABAW9Gi0DRz5kx99dVX+rd/+zfV1dVJkkJCQpSdna2cnJxWbRAAACAQtCg0WSwW/fGPf9QTTzyhTz75RB07dtS1114rm83W2v0BAAAEhBaFpiZXXHGFhg0b1lq9AAAABKwWbQQHAABobwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmBDwoalXr16yWCzNjszMTElScnJys7mpU6f6XKOyslJpaWnq1KmTIiMjNXPmTJ0+fdqnZufOnRoyZIhsNpt69+6tgoKCi7VEAADQBnTwdwPn88EHH6ihocE4Lysr0y9/+UuNGzfOGJs8ebLmzp1rnHfq1Mn4uqGhQWlpaYqOjtbu3bt1/Phx3Xvvvbrsssv0hz/8QZJUUVGhtLQ0TZ06VatXr1ZhYaEefPBBdevWTQ6H4yKsEgAABLqAD01du3b1OZ8/f76uueYa3XLLLcZYp06dFB0dfcbXb9++XR9//LF27NihqKgoDRo0SPPmzVN2drby8vJktVq1YsUKxcXFadGiRZKkfv36adeuXVq8ePFZQ1Ntba1qa2uNc4/H83OXCgAAAljAP577sbq6Ov31r3/VAw88IIvFYoyvXr1aXbp00cCBA5WTk6PvvvvOmHO5XIqPj1dUVJQx5nA45PF4dPDgQaMmJSXF570cDodcLtdZe8nPz1d4eLhxxMbGttYyAQBAAAr4O00/tnHjRp08eVL33XefMTZhwgT17NlTMTExOnDggLKzs1VeXq7XXntNkuR2u30CkyTj3O12n7PG4/Ho+++/V8eOHZv1kpOTo6ysLOPc4/EQnAAAuIS1qdD04osvavTo0YqJiTHGpkyZYnwdHx+vbt266bbbbtORI0d0zTXXXLBebDabbDbbBbs+AAAILG3m8dzf//537dixQw8++OA565KSkiRJhw8fliRFR0erqqrKp6bpvGkf1NlqwsLCzniXCQAAtD9tJjS99NJLioyMVFpa2jnrSktLJUndunWTJNntdn300Uc6ceKEUeN0OhUWFqb+/fsbNYWFhT7XcTqdstvtrbgCAADQlrWJ0NTY2KiXXnpJkyZNUocO//+J4pEjRzRv3jyVlJTo6NGjeuONN3TvvfdqxIgRSkhIkCSlpqaqf//+uueee7R//35t27ZNs2fPVmZmpvF4berUqfrss880a9YsHTp0SM8995xeeeUVzZgxwy/rBQAAgadNhKYdO3aosrJSDzzwgM+41WrVjh07lJqaqr59++rxxx/X2LFj9eabbxo1wcHB2rRpk4KDg2W323X33Xfr3nvv9flcp7i4OG3evFlOp1OJiYlatGiRXnjhBT6jCQAAGNrERvDU1FR5vd5m47GxsSoqKjrv63v27KktW7acsyY5OVn79u1rcY8AAODS1ibuNAEAAPgboQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwIaBDU15eniwWi8/Rt29fY/6HH35QZmamrrrqKl1xxRUaO3asqqqqfK5RWVmptLQ0derUSZGRkZo5c6ZOnz7tU7Nz504NGTJENptNvXv3VkFBwcVYHgAAaEMCOjRJ0oABA3T8+HHj2LVrlzE3Y8YMvfnmm1q/fr2Kiop07Ngx3XHHHcZ8Q0OD0tLSVFdXp927d+vll19WQUGBcnNzjZqKigqlpaVp5MiRKi0t1fTp0/Xggw9q27ZtF3WdAAAgsHXwdwPn06FDB0VHRzcbr66u1osvvqg1a9bo1ltvlSS99NJL6tevn/bs2aMbbrhB27dv18cff6wdO3YoKipKgwYN0rx585Sdna28vDxZrVatWLFCcXFxWrRokSSpX79+2rVrlxYvXiyHw3FR1woAAAJXwN9p+vTTTxUTE6Orr75aEydOVGVlpSSppKRE9fX1SklJMWr79u2rHj16yOVySZJcLpfi4+MVFRVl1DgcDnk8Hh08eNCo+fE1mmqarnE2tbW18ng8PgcAALh0BXRoSkpKUkFBgbZu3arnn39eFRUVGj58uL799lu53W5ZrVZ17tzZ5zVRUVFyu92SJLfb7ROYmuab5s5V4/F49P3335+1t/z8fIWHhxtHbGzsz10uAAAIYAH9eG706NHG1wkJCUpKSlLPnj31yiuvqGPHjn7sTMrJyVFWVpZx7vF4CE4AAFzCAvpO0z/r3LmzfvGLX+jw4cOKjo5WXV2dTp486VNTVVVl7IGKjo5u9tt0TefnqwkLCztnMLPZbAoLC/M5AADApatNhaZTp07pyJEj6tatm4YOHarLLrtMhYWFxnx5ebkqKytlt9slSXa7XR999JFOnDhh1DidToWFhal///5GzY+v0VTTdA0AAAApwEPTv//7v6uoqEhHjx7V7t279Zvf/EbBwcEaP368wsPDlZGRoaysLL399tsqKSnR/fffL7vdrhtuuEGSlJqaqv79++uee+7R/v37tW3bNs2ePVuZmZmy2WySpKlTp+qzzz7TrFmzdOjQIT333HN65ZVXNGPGDH8uHQAABJiA3tP0j3/8Q+PHj9dXX32lrl276uabb9aePXvUtWtXSdLixYsVFBSksWPHqra2Vg6HQ88995zx+uDgYG3atEkPP/yw7Ha7Lr/8ck2aNElz5841auLi4rR582bNmDFDS5cuVffu3fXCCy/wcQMAAMBHQIemtWvXnnM+JCREy5cv1/Lly89a07NnT23ZsuWc10lOTta+ffta1CMAAGgfAvrxHAAAQKAgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMCOjQlJ+fr2HDhik0NFSRkZFKT09XeXm5T01ycrIsFovPMXXqVJ+ayspKpaWlqVOnToqMjNTMmTN1+vRpn5qdO3dqyJAhstls6t27twoKCi708gAAQBsS0KGpqKhImZmZ2rNnj5xOp+rr65WamqqamhqfusmTJ+v48ePGsWDBAmOuoaFBaWlpqqur0+7du/Xyyy+roKBAubm5Rk1FRYXS0tI0cuRIlZaWavr06XrwwQe1bdu2i7ZWAAAQ2Dr4u4Fz2bp1q895QUGBIiMjVVJSohEjRhjjnTp1UnR09BmvsX37dn388cfasWOHoqKiNGjQIM2bN0/Z2dnKy8uT1WrVihUrFBcXp0WLFkmS+vXrp127dmnx4sVyOBwXboEAAKDNCOg7Tf+surpakhQREeEzvnr1anXp0kUDBw5UTk6OvvvuO2PO5XIpPj5eUVFRxpjD4ZDH49HBgweNmpSUFJ9rOhwOuVyus/ZSW1srj8fjcwAAgEtXQN9p+rHGxkZNnz5dN910kwYOHGiMT5gwQT179lRMTIwOHDig7OxslZeX67XXXpMkud1un8AkyTh3u93nrPF4PPr+++/VsWPHZv3k5+drzpw5rbpGAAAQuNpMaMrMzFRZWZl27drlMz5lyhTj6/j4eHXr1k233Xabjhw5omuuueaC9ZOTk6OsrCzj3OPxKDY29oK9HwAA8K828Xhu2rRp2rRpk95++2117979nLVJSUmSpMOHD0uSoqOjVVVV5VPTdN60D+psNWFhYWe8yyRJNptNYWFhPgcAALh0BXRo8nq9mjZtmjZs2KC33npLcXFx531NaWmpJKlbt26SJLvdro8++kgnTpwwapxOp8LCwtS/f3+jprCw0Oc6TqdTdru9lVYCAADauoAOTZmZmfrrX/+qNWvWKDQ0VG63W263W99//70k6ciRI5o3b55KSkp09OhRvfHGG7r33ns1YsQIJSQkSJJSU1PVv39/3XPPPdq/f7+2bdum2bNnKzMzUzabTZI0depUffbZZ5o1a5YOHTqk5557Tq+88opmzJjht7UDAIDAEtCh6fnnn1d1dbWSk5PVrVs341i3bp0kyWq1aseOHUpNTVXfvn31+OOPa+zYsXrzzTeNawQHB2vTpk0KDg6W3W7X3XffrXvvvVdz5841auLi4rR582Y5nU4lJiZq0aJFeuGFF/i4AQAAYAjojeBer/ec87GxsSoqKjrvdXr27KktW7acsyY5OVn79u37Sf0BAID2I6DvNAEAAAQKQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCQAAwARCEwAAgAmEJgAAABMITQAAACYQmgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAwgdAEAABgAqEJAADABELTP1m+fLl69eqlkJAQJSUl6f333/d3SwAAIAAQmn5k3bp1ysrK0pNPPqkPP/xQiYmJcjgcOnHihL9bAwAAfkZo+pGnn35akydP1v3336/+/ftrxYoV6tSpk1atWuXv1gAAgJ918HcDgaKurk4lJSXKyckxxoKCgpSSkiKXy9Wsvra2VrW1tcZ5dXW1JMnj8Vz4ZlvB6fof/N1Cm9VW/hu3NQ0/1J6/CGfFz+WFUVfDz2VLtZWfyaY+vV7veWsJTf/nyy+/VENDg6KionzGo6KidOjQoWb1+fn5mjNnTrPx2NjYC9YjAkP4hnx/twA0E/77+f5uAfCxUv/p7xZ+km+//Vbh4eHnrCE0tVBOTo6ysrKM88bGRn399de66qqrZLFY/NhZ2+fxeBQbG6vPP/9cYWFh/m4H4GcSAYefydbj9Xr17bffKiYm5ry1hKb/06VLFwUHB6uqqspnvKqqStHR0c3qbTabbDabz1jnzp0vZIvtTlhYGP8zQEDhZxKBhp/J1nG+O0xN2Aj+f6xWq4YOHarCwkJjrLGxUYWFhbLb7X7sDAAABALuNP1IVlaWJk2apOuuu07XX3+9lixZopqaGt1///3+bg0AAPgZoelH7rzzTn3xxRfKzc2V2+3WoEGDtHXr1mabw3Fh2Ww2Pfnkk80efwL+ws8kAg0/k/5h8Zr5HTsAAIB2jj1NAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQhICyfPly9erVSyEhIUpKStL777/v75bQjhUXF2vMmDGKiYmRxWLRxo0b/d0S2rn8/HwNGzZMoaGhioyMVHp6usrLy/3dVrtBaELAWLdunbKysvTkk0/qww8/VGJiohwOh06cOOHv1tBO1dTUKDExUcuXL/d3K4AkqaioSJmZmdqzZ4+cTqfq6+uVmpqqmpoaf7fWLvCRAwgYSUlJGjZsmJYtWybpfz+RPTY2Vo888oj+4z/+w8/dob2zWCzasGGD0tPT/d0KYPjiiy8UGRmpoqIijRgxwt/tXPK404SAUFdXp5KSEqWkpBhjQUFBSklJkcvl8mNnABC4qqurJUkRERF+7qR9IDQhIHz55ZdqaGho9unrUVFRcrvdfuoKAAJXY2Ojpk+frptuukkDBw70dzvtAn9GBQCANigzM1NlZWXatWuXv1tpNwhNCAhdunRRcHCwqqqqfMarqqoUHR3tp64AIDBNmzZNmzZtUnFxsbp37+7vdtoNHs8hIFitVg0dOlSFhYXGWGNjowoLC2W32/3YGQAEDq/Xq2nTpmnDhg166623FBcX5++W2hXuNCFgZGVladKkSbruuut0/fXXa8mSJaqpqdH999/v79bQTp06dUqHDx82zisqKlRaWqqIiAj16NHDj52hvcrMzNSaNWv0+uuvKzQ01NjzGR4ero4dO/q5u0sfHzmAgLJs2TItXLhQbrdbgwYN0jPPPKOkpCR/t4V2aufOnRo5cmSz8UmTJqmgoODiN4R2z2KxnHH8pZde0n333Xdxm2mHCE0AAAAmsKcJAADABEITAACACYQmAAAAEwhNAAAAJhCaAAAATCA0AQAAmEBoAgAAMIHQBAAAYAKhCUC7kZycrOnTp5uq3blzpywWi06ePPmz3rNXr15asmTJz7oGgMBAaAIAADCB0AQAAGACoQlAu/Rf//Vfuu666xQaGqro6GhNmDBBJ06caFb37rvvKiEhQSEhIbrhhhtUVlbmM79r1y4NHz5cHTt2VGxsrB599FHV1NRcrGUAuIgITQDapfr6es2bN0/79+/Xxo0bdfTo0TP+lfiZM2dq0aJF+uCDD9S1a1eNGTNG9fX1kqQjR45o1KhRGjt2rA4cOKB169Zp165dmjZt2kVeDYCLoYO/GwAAf3jggQeMr6+++mo988wzGjZsmE6dOqUrrrjCmHvyySf1y1/+UpL08ssvq3v37tqwYYN++9vfKj8/XxMnTjQ2l1977bV65plndMstt+j5559XSEjIRV0TgAuLO00A2qWSkhKNGTNGPXr0UGhoqG655RZJUmVlpU+d3W43vo6IiFCfPn30ySefSJL279+vgoICXXHFFcbhcDjU2NioioqKi7cYABcFd5oAtDs1NTVyOBxyOBxavXq1unbtqsrKSjkcDtXV1Zm+zqlTp/TQQw/p0UcfbTbXo0eP1mwZQAAgNAFodw4dOqSvvvpK8+fPV2xsrCRp7969Z6zds2ePEYC++eYb/e1vf1O/fv0kSUOGDNHHH3+s3r17X5zGAfgVj+cAtDs9evSQ1WrVs88+q88++0xvvPGG5s2bd8bauXPnqrCwUGVlZbrvvvvUpUsXpaenS5Kys7O1e/duTZs2TaWlpfr000/1+uuvsxEcuEQRmgC0O127dlVBQYHWr1+v/v37a/78+frTn/50xtr58+frscce09ChQ+V2u/Xmm2/KarVKkhISElRUVKS//e1vGj58uAYPHqzc3FzFxMRczOUAuEgsXq/X6+8mAAAAAh13mgAAAEwgNAEAAJhAaAIAADCB0AQAAGACoQkAAMAEQhMAAIAJhCYAAAATCE0AAAAmEJoAAABMIDQBAACYQGgCAAAw4f8BzIFn1njweRwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='label',data=df,palette='viridis',gap=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "964381c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da004fd07b224f6ebc99b6cf0513e184",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Generating train split: 0 examples [00:00, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset, Features, Value\n",
    "\n",
    "# Define a schema that matches your CSV file\n",
    "features = Features({\n",
    "    \"Unnamed: 0\": Value(\"int64\"), \n",
    "    \"text\": Value(\"string\"), \n",
    "    \"label\": Value(\"int64\")\n",
    "})\n",
    "\n",
    "# Load the dataset using your custom schema\n",
    "dataset = load_dataset('csv', data_files='new_train.csv', features=features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f45c7197",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.remove_columns('Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c68160d9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label'],\n",
       "        num_rows: 57477\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5db42030",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ckpt = 'distilbert-base-uncased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2eec8a41",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_ckpt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "21079bbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_tokenizer(batch):\n",
    "    return tokenizer(batch['text'],padding=True,truncation=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "15fef48b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4752e2a9b71a4176b9bd0a822de91dbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/57477 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset_encoded = dataset.map(apply_tokenizer,batched=True,batch_size=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3a11e4fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_encoded.set_format(\"pt\",columns=['input_ids','attention_mask','label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0881a3fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_encoded = dataset_encoded['train'].train_test_split(train_size=0.85)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "41f8c6d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['text', 'label', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 39083\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['text', 'label', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 6898\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "eafd62b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "num_labels = 3\n",
    "model = (AutoModelForSequenceClassification.from_pretrained(model_ckpt,num_labels=num_labels,).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1649504a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(pred):\n",
    "    labels = pred.label_ids\n",
    "    preds = pred.predictions.argmax(-1)\n",
    "    f1 = f1_score(labels,preds,average='weighted')\n",
    "    acc = accuracy_score(labels,preds)\n",
    "    return {'f1':f1,'acc':acc}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "8582dfe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "new_model='LLM_PROMPTS_CLASSIFICATION'\n",
    "batch_size=4\n",
    "logging_steps = len(dataset_encoded['train'])//batch_size\n",
    "num_train_epochs = 4\n",
    "training_args = TrainingArguments(output_dir=new_model,\n",
    "                                 per_device_train_batch_size=batch_size,\n",
    "                                 per_device_eval_batch_size=batch_size,\n",
    "                                 log_level=\"error\",\n",
    "                                 logging_steps=logging_steps,\n",
    "                                 eval_strategy=\"epoch\",\n",
    "                                 disable_tqdm=False,\n",
    "                                 weight_decay=0.01,\n",
    "                                 learning_rate=2e-5,\n",
    "                                 push_to_hub=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "ad3faf64",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Trainer\n",
    "trainer = Trainer(model=model,\n",
    "                 args=training_args,\n",
    "                 compute_metrics=compute_metrics,\n",
    "                 train_dataset=dataset_encoded['train'],\n",
    "                 eval_dataset=dataset_encoded['test'],\n",
    "                 tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "016656d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='18' max='29313' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [   18/29313 00:04 < 2:17:47, 3.54 it/s, Epoch 0.00/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[43], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/anaconda3/envs/myenv/lib/python3.10/site-packages/transformers/trainer.py:2123\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   2121\u001b[0m         hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n\u001b[1;32m   2122\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 2123\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43minner_training_loop\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   2124\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2125\u001b[0m \u001b[43m        \u001b[49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresume_from_checkpoint\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2126\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtrial\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtrial\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2127\u001b[0m \u001b[43m        \u001b[49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mignore_keys_for_eval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   2128\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/anaconda3/envs/myenv/lib/python3.10/site-packages/transformers/trainer.py:2486\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2480\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m context():\n\u001b[1;32m   2481\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_step(model, inputs, num_items_in_batch)\n\u001b[1;32m   2483\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   2484\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   2485\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[0;32m-> 2486\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43misinf\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtr_loss_step\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   2487\u001b[0m ):\n\u001b[1;32m   2488\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   2489\u001b[0m     tr_loss \u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m+\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n\u001b[1;32m   2490\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e2db6d6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}