examples/web_extractor_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from example_config import SECRET_KEY\n",
    "openai.api_key = SECRET_KEY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from knowledgegpt.extractors.web_scrape_extractor import WebScrapeExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://en.wikipedia.org/wiki/Bombard_(weapon)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping website...\n",
      "Computing embeddings...\n",
      "model_lang en\n",
      "Selected 9 document sections:\n",
      "0\n",
      "33\n",
      "62\n",
      "60\n",
      "46\n",
      "67\n",
      "50\n",
      "61\n",
      "72\n",
      "Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don't know.\"\n",
      "\n",
      "Context:\n",
      "\n",
      "* Bombard (weapon) - Wikipedia\n",
      "* Bombard (weapon)\n",
      "* Wikimedia Commons has media related to Bombards (weapon).\n",
      "* Bombard in its siege position, Denmark.\n",
      "* Terminology[edit] The term \"bombard\" was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries.[2] Despite its strong association with large cannons, there is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.[3]\n",
      "* ^ Gwei-Djen, Lu; Needham, Joseph; Chi-Hsing, Phan (July 1988). \"The Oldest Representation of a Bombard\". Technology and Culture. 29 (3): 594–605. doi:10.2307/3105275. JSTOR 3105275.\n",
      "* \"Hand bombard\", 1390–1400\n",
      "* A bombard recovered from the well of Cardiff castle, in swivel configuration.\n",
      "* ^ \"Bodiam Bombard | Kent and Sussex Courier\". Archived from the original on 2015-09-23. Retrieved 2015-04-13.\n",
      "\n",
      " Q: What is a bombard?\n",
      " A:\n",
      "all_done!\n"
     ]
    }
   ],
   "source": [
    "scrape_website = WebScrapeExtractor( url=url, embedding_extractor=\"hf\", model_lang=\"en\", is_turbo=True)\n",
    "answer, prompt, messages = scrape_website.extract(query=\"What is a bombard?\",max_tokens=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A bombard is a type of weapon that was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries. There is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping website...\n",
      "Computing embeddings...\n",
      "not the first time\n",
      "Answering query...\n",
      "What is a bombard?\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "answer, prompt, messages = scrape_website.extract(query=\"What is a bombard?\",max_tokens=300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A bombard is a type of weapon that was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries. There is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'role': 'system', 'content': 'you are a helpful assistant'},\n",
       " {'role': 'user',\n",
       "  'content': 'Answer the question as truthfully as possible using the provided context, and if the answer is not contained within the text below, say \"I don\\'t know.\"\\n\\nContext:\\n\\n* Bombard (weapon) - Wikipedia\\n* Bombard (weapon)\\n* Wikimedia Commons has media related to Bombards (weapon).\\n* Bombard in its siege position, Denmark.\\n* Terminology[edit] The term \"bombard\" was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries.[2] Despite its strong association with large cannons, there is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.[3]\\n* ^ Gwei-Djen, Lu; Needham, Joseph; Chi-Hsing, Phan (July 1988). \"The Oldest Representation of a Bombard\". Technology and Culture. 29 (3): 594–605. doi:10.2307/3105275. JSTOR\\xa03105275.\\n* \"Hand bombard\", 1390–1400\\n* A bombard recovered from the well of Cardiff castle, in swivel configuration.\\n* ^ \"Bodiam Bombard | Kent and Sussex Courier\". Archived from the original on 2015-09-23. Retrieved 2015-04-13.\\n\\n Q: What is a bombard?\\n A:'},\n",
       " {'role': 'assistant',\n",
       "  'content': 'A bombard is a type of weapon that was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries. There is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.'},\n",
       " {'role': 'user', 'content': 'What is a bombard?'},\n",
       " {'role': 'assistant',\n",
       "  'content': 'A bombard is a type of weapon that was first used to describe guns of any kind from the early to mid-14th century, but it was later applied primarily to large cannons during the 14th to 15th centuries. There is no standard size for bombards, and the term has been applied to cannons only a meter in length as well as cannons several meters long weighing up to 20 tonnes.'}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "knowledgegpt-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}