Fertilizer_Prediction-E2E-V4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "a83fec55",
   "metadata": {},
   "outputs": [],
   "source": [
    "# installs google-api-python-client, google-auth, google-auth-oauthlib\n",
    "# !pip install --upgrade google-api-python-client google-auth google-auth-oauthlib google-auth-httplib2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "1a2a1042",
   "metadata": {},
   "outputs": [],
   "source": [
    "# updates pip\n",
    "# pip install --upgrade pip\n",
    "# # restart kernel once updates are done."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "de2d435d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: google-api-python-client==2.98.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 1)) (2.98.0)\n",
      "Requirement already satisfied: google-auth==2.23.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 2)) (2.23.0)\n",
      "Requirement already satisfied: google-auth-oauthlib==1.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 3)) (1.0.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 4)) (2.2.2)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 5)) (3.9.2)\n",
      "Requirement already satisfied: numpy in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 6)) (1.26.4)\n",
      "Requirement already satisfied: seaborn==0.12.2 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 7)) (0.12.2)\n",
      "Requirement already satisfied: scikit-learn==1.3.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 8)) (1.3.1)\n",
      "Requirement already satisfied: xgboost==1.7.6 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 9)) (1.7.6)\n",
      "Requirement already satisfied: tensorflow in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from -r requirements.txt (line 10)) (2.18.0)\n",
      "Requirement already satisfied: httplib2<1.dev0,>=0.15.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-python-client==2.98.0->-r requirements.txt (line 1)) (0.22.0)\n",
      "Requirement already satisfied: google-auth-httplib2>=0.1.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-python-client==2.98.0->-r requirements.txt (line 1)) (0.2.0)\n",
      "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-python-client==2.98.0->-r requirements.txt (line 1)) (2.22.0)\n",
      "Requirement already satisfied: uritemplate<5,>=3.0.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-python-client==2.98.0->-r requirements.txt (line 1)) (4.1.1)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-auth==2.23.0->-r requirements.txt (line 2)) (5.3.3)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-auth==2.23.0->-r requirements.txt (line 2)) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-auth==2.23.0->-r requirements.txt (line 2)) (4.9)\n",
      "Requirement already satisfied: urllib3<2.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-auth==2.23.0->-r requirements.txt (line 2)) (1.26.20)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-auth-oauthlib==1.0.0->-r requirements.txt (line 3)) (2.0.0)\n",
      "Requirement already satisfied: scipy>=1.5.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from scikit-learn==1.3.1->-r requirements.txt (line 8)) (1.13.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from scikit-learn==1.3.1->-r requirements.txt (line 8)) (1.4.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from scikit-learn==1.3.1->-r requirements.txt (line 8)) (3.5.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from pandas->-r requirements.txt (line 4)) (2.9.0.post0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from pandas->-r requirements.txt (line 4)) (2024.1)\n",
      "Requirement already satisfied: tzdata>=2022.7 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from pandas->-r requirements.txt (line 4)) (2023.3)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (1.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (4.51.0)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (24.1)\n",
      "Requirement already satisfied: pillow>=8 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (10.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from matplotlib->-r requirements.txt (line 5)) (3.1.2)\n",
      "Requirement already satisfied: tensorflow-intel==2.18.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow->-r requirements.txt (line 10)) (2.18.0)\n",
      "Requirement already satisfied: absl-py>=1.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.1.0)\n",
      "Requirement already satisfied: astunparse>=1.6.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (1.6.3)\n",
      "Requirement already satisfied: flatbuffers>=24.3.25 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (24.3.25)\n",
      "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.6.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.2.0)\n",
      "Requirement already satisfied: libclang>=13.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (18.1.1)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.4.0)\n",
      "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (4.25.3)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.32.3)\n",
      "Requirement already satisfied: setuptools in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (75.1.0)\n",
      "Requirement already satisfied: six>=1.12.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (1.16.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.5.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.6 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (4.11.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (1.14.1)\n",
      "Requirement already satisfied: grpcio<2.0,>=1.24.3 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (1.67.1)\n",
      "Requirement already satisfied: tensorboard<2.19,>=2.18 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.18.0)\n",
      "Requirement already satisfied: keras>=3.5.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.6.0)\n",
      "Requirement already satisfied: h5py>=3.11.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.11.0)\n",
      "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.4.1)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client==2.98.0->-r requirements.txt (line 1)) (1.65.0)\n",
      "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.3 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0.dev0,>=1.31.5->google-api-python-client==2.98.0->-r requirements.txt (line 1)) (1.25.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth==2.23.0->-r requirements.txt (line 2)) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib==1.0.0->-r requirements.txt (line 3)) (3.2.2)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from astunparse>=1.6.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.44.0)\n",
      "Requirement already satisfied: rich in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (13.7.1)\n",
      "Requirement already satisfied: namex in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.0.8)\n",
      "Requirement already satisfied: optree in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.13.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2024.8.30)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorboard<2.19,>=2.18->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.4.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorboard<2.19,>=2.18->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.7.2)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from tensorboard<2.19,>=2.18->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (3.0.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.1.3)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from rich->keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.2.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from rich->keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (2.15.1)\n",
      "Requirement already satisfied: mdurl~=0.1 in c:\\users\\ssankati\\appdata\\local\\anaconda\\lib\\site-packages (from markdown-it-py>=2.2.0->rich->keras>=3.5.0->tensorflow-intel==2.18.0->tensorflow->-r requirements.txt (line 10)) (0.1.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "074e3988",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from googleapiclient.discovery import build\n",
    "from googleapiclient.http import MediaIoBaseDownload\n",
    "from google.oauth2.credentials import Credentials\n",
    "from google_auth_oauthlib.flow import InstalledAppFlow\n",
    "from io import BytesIO\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import sklearn.metrics as metrics\n",
    "from xgboost import XGBClassifier\n",
    "import pickle as pkl\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer  # Importing ColumnTransformer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.metrics import accuracy_score, classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "58393403",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(filename='input.txt'):\n",
    "    config={}\n",
    "    with open(filename, 'r') as file:\n",
    "        for line in file:\n",
    "            key, value = line.strip().split('=')\n",
    "            config[key] = value\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2ecfd510",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_details = load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "61e4eb18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " file id is 1ZY9Qv5nmDJ0yzffr5qCdHPrWMfbiBf5t\n"
     ]
    }
   ],
   "source": [
    "file_id = input_details['file_id']\n",
    "print(f' file id is {file_id}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "224b0f32",
   "metadata": {},
   "outputs": [],
   "source": [
    "def access_data_from_drive(file_id):\n",
    "    # Define scope for accessing Google Drive\n",
    "    SCOPES = ['https://www.googleapis.com/auth/drive.readonly']\n",
    "\n",
    "    #  Authenticate using OAuth2 credentials\n",
    "    flow = InstalledAppFlow.from_client_secrets_file('./credentials.json', SCOPES)\n",
    "    creds = flow.run_local_server(port=0)\n",
    "\n",
    "    # Build the Google Drive service\n",
    "    service = build('drive','v3',credentials=creds)\n",
    "\n",
    "    # File ID from shareable link\n",
    "    file_id = file_id#'1ZY9Qv5nmDJ0yzffr5qCdHPrWMfbiBf5t'\n",
    "\n",
    "    # Reuqest file metadata\n",
    "    file_metadata = service.files().get(fileId=file_id).execute()\n",
    "    print(\"File Metadata\", file_metadata)\n",
    "\n",
    "    # Read the file content into memory\n",
    "    file_content = BytesIO()\n",
    "\n",
    "    # Request the media content from Google Drive\n",
    "    request = service.files().get_media(fileId=file_id)\n",
    "    media_downloader = MediaIoBaseDownload(file_content, request)\n",
    "\n",
    "    # Download the file content into the buffer\n",
    "    done = False\n",
    "    while not done:\n",
    "        status, done = media_downloader.next_chunk()\n",
    "        print(f\"Download {int(status.progress()*100)}% complete.\")\n",
    "\n",
    "\n",
    "    # Move the buffer's position back to the start\n",
    "    file_content.seek(0)\n",
    "\n",
    "    # Process the file content\n",
    "    df = pd.read_csv(file_content)\n",
    "    print(df)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d83886d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Please visit this URL to authorize this application: https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=970406091830-342nl3nqlck6q0mscvg8vj0fnm3r36ot.apps.googleusercontent.com&redirect_uri=http%3A%2F%2Flocalhost%3A57909%2F&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.readonly&state=ewlytpMXllirhTyojEqhdMxzbAkWsr&access_type=offline\n",
      "File Metadata {'kind': 'drive#file', 'id': '1ZY9Qv5nmDJ0yzffr5qCdHPrWMfbiBf5t', 'name': 'Fertilizer_Prediction.csv', 'mimeType': 'text/csv'}\n",
      "Download 100% complete.\n",
      "    Temparature  Humidity   Moisture Soil Type  Crop Type  Nitrogen  \\\n",
      "0            26         52        38     Sandy      Maize        37   \n",
      "1            29         52        45     Loamy  Sugarcane        12   \n",
      "2            34         65        62     Black     Cotton         7   \n",
      "3            32         62        34       Red    Tobacco        22   \n",
      "4            28         54        46    Clayey      Paddy        35   \n",
      "..          ...        ...       ...       ...        ...       ...   \n",
      "94           25         50        32    Clayey     Pulses        24   \n",
      "95           30         60        27       Red    Tobacco         4   \n",
      "96           38         72        51     Loamy      Wheat        39   \n",
      "97           36         60        43     Sandy    Millets        15   \n",
      "98           29         58        57     Black  Sugarcane        12   \n",
      "\n",
      "    Potassium  Phosphorous Fertilizer Name  \n",
      "0           0            0            Urea  \n",
      "1           0           36             DAP  \n",
      "2           9           30        14-35-14  \n",
      "3           0           20           28-28  \n",
      "4           0            0            Urea  \n",
      "..        ...          ...             ...  \n",
      "94          0           19           28-28  \n",
      "95         17           17        10-26-26  \n",
      "96          0            0            Urea  \n",
      "97          0           41             DAP  \n",
      "98          0           10           20-20  \n",
      "\n",
      "[99 rows x 9 columns]\n"
     ]
    }
   ],
   "source": [
    "df = access_data_from_drive(file_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "1556fb2a-ef22-4c8f-8cb7-ae6ff320b0d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_overview(df):\n",
    "    \n",
    "    # print list of columns\n",
    "    print('List of columns in teh given data are: \\n', df.columns)\n",
    "    \n",
    "    # change the name of columns to lowercase & replace space with '_'\n",
    "    df.columns = df.columns.str.strip().str.lower().str.replace(' ','_')\n",
    "    print('\\n List of columns in teh given data are: \\n', df.columns)\n",
    "    \n",
    "    # details of the data\n",
    "    print('\\n Shape of the data is: \\n',df.shape)\n",
    "    print('\\n Data information is: \\n', df.info())\n",
    "    print('\\n Five point summary of the data is: \\n', df.describe().T)\n",
    "    print('\\n Number of NA values in the data: \\n', df.isna().sum())\n",
    "    print('\\n Number of Null values in the data: \\n', df.isnull().sum())\n",
    "    print('\\n Number of duplicated records are : \\n', df.duplicated().sum())\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "772c6c86-2f85-43e7-9a15-b545cd0878f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of columns in teh given data are: \n",
      " Index(['Temparature', 'Humidity ', 'Moisture', 'Soil Type', 'Crop Type',\n",
      "       'Nitrogen', 'Potassium', 'Phosphorous', 'Fertilizer Name'],\n",
      "      dtype='object')\n",
      "\n",
      " List of columns in teh given data are: \n",
      " Index(['temparature', 'humidity', 'moisture', 'soil_type', 'crop_type',\n",
      "       'nitrogen', 'potassium', 'phosphorous', 'fertilizer_name'],\n",
      "      dtype='object')\n",
      "\n",
      " Shape of the data is: \n",
      " (99, 9)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 99 entries, 0 to 98\n",
      "Data columns (total 9 columns):\n",
      " #   Column           Non-Null Count  Dtype \n",
      "---  ------           --------------  ----- \n",
      " 0   temparature      99 non-null     int64 \n",
      " 1   humidity         99 non-null     int64 \n",
      " 2   moisture         99 non-null     int64 \n",
      " 3   soil_type        99 non-null     object\n",
      " 4   crop_type        99 non-null     object\n",
      " 5   nitrogen         99 non-null     int64 \n",
      " 6   potassium        99 non-null     int64 \n",
      " 7   phosphorous      99 non-null     int64 \n",
      " 8   fertilizer_name  99 non-null     object\n",
      "dtypes: int64(6), object(3)\n",
      "memory usage: 7.1+ KB\n",
      "\n",
      " Data information is: \n",
      " None\n",
      "\n",
      " Five point summary of the data is: \n",
      "              count       mean        std   min   25%   50%   75%   max\n",
      "temparature   99.0  30.282828   3.502304  25.0  28.0  30.0  33.0  38.0\n",
      "humidity      99.0  59.151515   5.840331  50.0  54.0  60.0  64.0  72.0\n",
      "moisture      99.0  43.181818  11.271568  25.0  34.0  41.0  50.5  65.0\n",
      "nitrogen      99.0  18.909091  11.599693   4.0  10.0  13.0  24.0  42.0\n",
      "potassium     99.0   3.383838   5.814667   0.0   0.0   0.0   7.5  19.0\n",
      "phosphorous   99.0  18.606061  13.476978   0.0   9.0  19.0  30.0  42.0\n",
      "\n",
      " Number of NA values in the data: \n",
      " temparature        0\n",
      "humidity           0\n",
      "moisture           0\n",
      "soil_type          0\n",
      "crop_type          0\n",
      "nitrogen           0\n",
      "potassium          0\n",
      "phosphorous        0\n",
      "fertilizer_name    0\n",
      "dtype: int64\n",
      "\n",
      " Number of Null values in the data: \n",
      " temparature        0\n",
      "humidity           0\n",
      "moisture           0\n",
      "soil_type          0\n",
      "crop_type          0\n",
      "nitrogen           0\n",
      "potassium          0\n",
      "phosphorous        0\n",
      "fertilizer_name    0\n",
      "dtype: int64\n",
      "\n",
      " Number of duplicated records are : \n",
      " 0\n"
     ]
    }
   ],
   "source": [
    "df = data_overview(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "84b1ff6d-2639-4707-b893-065a9f84e633",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom transformer for handling NaN or Null checking\n",
    "class NullChecker(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        self.null_columns_ = X.columns[X.isnull().any()].tolist()\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X = X.fillna(0)  # Replace NaN with 0 or any other strategy\n",
    "        return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b97cf884-e118-4c88-979b-fe3ac87aaf9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom transformer for label encoding\n",
    "class LabelEncoderTransformer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        print(\"=== Initiating label encoding ===\")\n",
    "        self.le = LabelEncoder()\n",
    "        self.obj_col_list = X.select_dtypes(include=['object']).columns.tolist()\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X_encoded = X.copy()\n",
    "        for col in self.obj_col_list:\n",
    "            X_encoded[col] = self.le.fit_transform(X[col])\n",
    "        print(\"=== Label encoding is completed ===\")\n",
    "        return X_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cbd9cf60-291a-42fd-ad39-1668b6d3f437",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Custom transformer for standard scaling\n",
    "class StandardScalerTransformer(BaseEstimator, TransformerMixin):\n",
    "    def fit(self, X, y=None):\n",
    "        print(\"=== Initiating standard scaling ===\")\n",
    "        self.scaler = StandardScaler()\n",
    "        self.num_col_list = X.select_dtypes(include=['int64', 'float64']).columns.tolist()\n",
    "        self.scaler.fit(X[self.num_col_list])\n",
    "        print(\"=== Standard scaling is completed ===\")\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        X_scaled = X.copy()\n",
    "        X_scaled[self.num_col_list] = self.scaler.transform(X[self.num_col_list])\n",
    "        return X_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "21047f38-f198-41f2-84b8-7700c36971c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitter stage as a transformer\n",
    "class DataSplitter(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, target_feature, test_size=0.25, random_state=1):\n",
    "        self.target_feature = target_feature\n",
    "        self.test_size = test_size\n",
    "        self.random_state = random_state\n",
    "    \n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        y = X[self.target_feature]\n",
    "        X = X.drop(columns=[self.target_feature])\n",
    "        return train_test_split(X, y, test_size=self.test_size, random_state=self.random_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "ffcab0f5-7794-4291-a4e3-9dec3a551789",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Model training and prediction as a final stage in the pipeline\n",
    "# class ModelTrainer(BaseEstimator, TransformerMixin):\n",
    "#     def __init__(self, model='RandomForest'):\n",
    "#         self.model = RandomForestClassifier(n_estimators=100)# if model == 'RandomForest' else XGBClassifier()\n",
    "    \n",
    "#     def fit(self, X, y):\n",
    "#         self.model.fit(X, y)\n",
    "#         return self\n",
    "    \n",
    "#     def predict(self, X):\n",
    "#         return self.model.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "c7913506-f2c2-47dc-8ed2-144e437ff4ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "class ModelTrainer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, \n",
    "                 n_estimators=100,\n",
    "                 max_depth=None,\n",
    "                 min_samples_split=2,\n",
    "                 min_samples_leaf=1,\n",
    "                 criterion='gini',\n",
    "                 bootstrap=True,\n",
    "                 model='RandomForest'):\n",
    "        \n",
    "        # Initialize the RandomForestClassifier with the provided parameters\n",
    "        self.model = RandomForestClassifier(\n",
    "            n_estimators=n_estimators,\n",
    "            max_depth=max_depth,\n",
    "            min_samples_split=min_samples_split,\n",
    "            min_samples_leaf=min_samples_leaf,\n",
    "            criterion=criterion,\n",
    "            bootstrap=bootstrap\n",
    "        ) if model == 'RandomForest' else XGBClassifier()\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        self.model.fit(X, y)\n",
    "        return self\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return self.model.predict(X)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a522e60e-67e1-486d-8a09-064bfa1ffcad",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model saving transformer\n",
    "class ModelSaver(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y):\n",
    "        # Save the model to a file\n",
    "        with open(self.filename, \"wb\") as file:\n",
    "            pkl.dump(y, file)  # Save the model (predictions) to file\n",
    "        return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "abf302f7-72d1-490d-a959-1106baf1022e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparing the pipeline\n",
    "pipeline = Pipeline([\n",
    "    ('null_checker', NullChecker()),\n",
    "    ('label_encoder', LabelEncoderTransformer()),  # Add label encoding as a stage\n",
    "    ('scaling', StandardScalerTransformer()),  # Add standard scaling as a stage\n",
    "    ('splitter', DataSplitter(target_feature='fertilizer_name')),\n",
    "    # ('model_trainer', ModelTrainer(model='RandomForest')),  # Change to 'XGBoost' if needed\n",
    "    ('model_trainer', ModelTrainer(n_estimators=50, max_depth=10, \n",
    "                                   min_samples_split=2, min_samples_leaf=1, \n",
    "                                   criterion='gini', bootstrap=True)),\n",
    "    ('model_saver', ModelSaver(filename='final_rf_model.pkl'))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "1f3b5304-9dac-45d3-babb-c450ecaa0500",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== Initiating label encoding ===\n",
      "=== Label encoding is completed ===\n",
      "=== Initiating standard scaling ===\n",
      "=== Standard scaling is completed ===\n"
     ]
    }
   ],
   "source": [
    "# Step 3: Fit the pipeline using the entire DataFrame\n",
    "pipeline.named_steps['null_checker'].fit(df)\n",
    "df_transformed = pipeline.named_steps['null_checker'].transform(df)\n",
    "\n",
    "# Encoding, Scaling, and Splitting\n",
    "df_transformed = pipeline.named_steps['label_encoder'].fit_transform(df_transformed)\n",
    "df_transformed = pipeline.named_steps['scaling'].fit_transform(df_transformed)\n",
    "\n",
    "# Split the dataset into training and testing sets\n",
    "X_train, X_test, y_train, y_test = pipeline.named_steps['splitter'].fit_transform(df_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "02c9af31-2754-4d39-8ff5-6e439d6a0c57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6 3 6 1 0 4 6 6 0 1 1 1 2 3 4 5 2 6 6 5 1 5 5 4 5]\n",
      "Accuracy: 0.96\n",
      "Classification Report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      0.67      0.80         3\n",
      "           1       0.80      1.00      0.89         4\n",
      "           2       1.00      1.00      1.00         2\n",
      "           3       1.00      1.00      1.00         2\n",
      "           4       1.00      1.00      1.00         3\n",
      "           5       1.00      1.00      1.00         5\n",
      "           6       1.00      1.00      1.00         6\n",
      "\n",
      "    accuracy                           0.96        25\n",
      "   macro avg       0.97      0.95      0.96        25\n",
      "weighted avg       0.97      0.96      0.96        25\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Step 4: Fit the model\n",
    "pipeline.named_steps['model_trainer'].fit(X_train, y_train)\n",
    "\n",
    "# Step 5: Save the model (after training)\n",
    "pipeline.named_steps['model_saver'].transform(X_test, pipeline.named_steps['model_trainer'].model)\n",
    "\n",
    "# Step 6: Make predictions using the test data\n",
    "y_pred = pipeline.named_steps['model_trainer'].predict(X_test)\n",
    "print(y_pred)\n",
    "\n",
    "# Evaluate\n",
    "print(\"Accuracy:\", accuracy_score(y_test, y_pred))\n",
    "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac63fe4d-e424-4353-b27d-2a2989435b71",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}