Extract_Information_From_INVOICE_using_GEMINI.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_2IevCj2_2x0"
      },
      "outputs": [],
      "source": [
        "!pip install -q -U google-generativeai"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import google.generativeai as genai"
      ],
      "metadata": {
        "id": "t1O3Je_ENtpf"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## SET API KEY"
      ],
      "metadata": {
        "id": "LWXvnZL8OHxy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Used to securely store your API key\n",
        "from google.colab import userdata\n",
        "\n",
        "GOOGLE_API_KEY=userdata.get('GOOGLE_API_KEY')\n",
        "\n",
        "genai.configure(api_key=GOOGLE_API_KEY)"
      ],
      "metadata": {
        "id": "rHXNl6qnN3qO"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## LIST OF MODELS"
      ],
      "metadata": {
        "id": "Sbj_mjeyOAG1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for m in genai.list_models():\n",
        "  if 'generateContent' in m.supported_generation_methods:\n",
        "    print(m.name)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "id": "GzWKJAOZN7Bt",
        "outputId": "dae637bc-e6f3-4519-820c-841dc0a71536"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "models/gemini-pro\n",
            "models/gemini-pro-vision\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Model Configuration\n",
        "MODEL_CONFIG = {\n",
        "  \"temperature\": 0.2,\n",
        "  \"top_p\": 1,\n",
        "  \"top_k\": 32,\n",
        "  \"max_output_tokens\": 4096,\n",
        "}\n",
        "\n",
        "## Safety Settings of Model\n",
        "safety_settings = [\n",
        "  {\n",
        "    \"category\": \"HARM_CATEGORY_HARASSMENT\",\n",
        "    \"threshold\": \"BLOCK_MEDIUM_AND_ABOVE\"\n",
        "  },\n",
        "  {\n",
        "    \"category\": \"HARM_CATEGORY_HATE_SPEECH\",\n",
        "    \"threshold\": \"BLOCK_MEDIUM_AND_ABOVE\"\n",
        "  },\n",
        "  {\n",
        "    \"category\": \"HARM_CATEGORY_SEXUALLY_EXPLICIT\",\n",
        "    \"threshold\": \"BLOCK_MEDIUM_AND_ABOVE\"\n",
        "  },\n",
        "  {\n",
        "    \"category\": \"HARM_CATEGORY_DANGEROUS_CONTENT\",\n",
        "    \"threshold\": \"BLOCK_MEDIUM_AND_ABOVE\"\n",
        "  }\n",
        "]"
      ],
      "metadata": {
        "id": "sErUXoOIOMKs"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "AkrCNMO9Oppr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## LOAD GEMINI MODEL WITH MODEL CONFIGURATIONS"
      ],
      "metadata": {
        "id": "M0ax-c4kOqE0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = genai.GenerativeModel(model_name = \"gemini-pro-vision\",\n",
        "                              generation_config = MODEL_CONFIG,\n",
        "                              safety_settings = safety_settings)"
      ],
      "metadata": {
        "id": "kDhL4GY1OuW_"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## DEFINE IMAGE FORMAT TO INPUT IN GEMINI"
      ],
      "metadata": {
        "id": "Stc93TB3Qzni"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pathlib import Path\n",
        "\n",
        "def image_format(image_path):\n",
        "    img = Path(image_path)\n",
        "\n",
        "    if not img.exists():\n",
        "        raise FileNotFoundError(f\"Could not find image: {img}\")\n",
        "\n",
        "    image_parts = [\n",
        "        {\n",
        "            \"mime_type\": \"image/png\", ## Mime type are PNG - image/png. JPEG - image/jpeg. WEBP - image/webp\n",
        "            \"data\": img.read_bytes()\n",
        "        }\n",
        "    ]\n",
        "    return image_parts\n"
      ],
      "metadata": {
        "id": "Eh_FdZ_nO2Xx"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## GEMINI MODEL OUTPUT"
      ],
      "metadata": {
        "id": "Xau7_2BuQ8ux"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def gemini_output(image_path, system_prompt, user_prompt):\n",
        "\n",
        "    image_info = image_format(image_path)\n",
        "    input_prompt= [system_prompt, image_info[0], user_prompt]\n",
        "    response = model.generate_content(input_prompt)\n",
        "    return response.text"
      ],
      "metadata": {
        "id": "gkDwnC9NQKSd"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### EXTRACTING PART OF THE INFORMATION FROM INVOICE"
      ],
      "metadata": {
        "id": "uDGItZ90Re2z"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "system_prompt = \"\"\"\n",
        "               You are a specialist in comprehending receipts.\n",
        "               Input images in the form of receipts will be provided to you,\n",
        "               and your task is to respond to questions based on the content of the input image.\n",
        "               \"\"\"\n",
        "\n",
        "image_path = \"/content/handwritten.png\"\n",
        "\n",
        "user_prompt = \"What is the balance amount in the image?\"\n",
        "\n",
        "gemini_output(image_path, system_prompt, user_prompt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "t5Amymj_Quw0",
        "outputId": "87d90a1f-0cfe-4157-9d27-d3cb6c753431"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "' The balance amount is 370.00.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## EXTRACTING WHOLE DATA IN JSON FROM INVOICE"
      ],
      "metadata": {
        "id": "c1tPGdtfR_sL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "system_prompt = \"\"\"\n",
        "               You are a specialist in comprehending receipts.\n",
        "               Input images in the form of receipts will be provided to you,\n",
        "               and your task is to respond to questions based on the content of the input image.\n",
        "               \"\"\"\n",
        "#system_prompt = \"Convert Invoice data into json format with appropriate json tags as required for the data in image \"\n",
        "image_path = \"/content/handwritten.png\"\n",
        "user_prompt = \"Convert Invoice data into json format with appropriate json tags as required for the data in image \""
      ],
      "metadata": {
        "id": "QTxNK6EDSUfg"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "output = gemini_output(image_path, system_prompt, user_prompt)"
      ],
      "metadata": {
        "id": "p6jDPxZITOv1"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Markdown\n",
        "Markdown(output)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 798
        },
        "id": "aOf5m0YWTQ52",
        "outputId": "45cc6e78-27ac-4116-915e-d7677966a74f"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ],
            "text/markdown": " ```json\n{\n  \"invoice_number\": \"8841\",\n  \"invoice_date\": \"19/12/05\",\n  \"customer_name\": \"M. Kennedy\",\n  \"items\": [\n    {\n      \"description\": \"Brado 42 Black\",\n      \"quantity\": 1,\n      \"price\": 500\n    },\n    {\n      \"description\": \"1 1/2\\\" smaller band\",\n      \"quantity\": 1,\n      \"price\": 0\n    },\n    {\n      \"description\": \"3/4\\\" bigger neck\",\n      \"quantity\": 1,\n      \"price\": 0\n    },\n    {\n      \"description\": \"Lemons collar\",\n      \"quantity\": 1,\n      \"price\": 0\n    },\n    {\n      \"description\": \"No epaulettes\",\n      \"quantity\": 1,\n      \"price\": 0\n    },\n    {\n      \"description\": \"Action Back\",\n      \"quantity\": 1,\n      \"price\": 0\n    },\n    {\n      \"description\": \"2 inside pockets with zips\",\n      \"quantity\": 1,\n      \"price\": 20\n    }\n  ],\n  \"total\": 570,\n  \"deposit\": 200,\n  \"balance\": 370\n}\n```"
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "O21AfbElTfTm"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}