diff --git a/solution.ipynb b/solution.ipynb new file mode 100644 index 0000000..3815efe --- /dev/null +++ b/solution.ipynb @@ -0,0 +1,947 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "7d8dc25a", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/dahliaelbanhawy/Desktop/IronHack/week_7/labs/lab-agent-vector-store/.venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n", + " warnings.warn(\n", + "USER_AGENT environment variable not set, consider setting it to identify your requests.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… All imports successful!\n" + ] + } + ], + "source": [ + "from langchain.chains import RetrievalQA\n", + "from langchain_community.vectorstores import Chroma\n", + "from langchain_openai import OpenAI, OpenAIEmbeddings\n", + "from langchain_text_splitters import CharacterTextSplitter\n", + "from langchain_community.document_loaders import TextLoader, WebBaseLoader\n", + "from langchain.agents import AgentType, Tool, initialize_agent\n", + "from langchain.memory import ConversationBufferMemory\n", + "import os\n", + "from dotenv import load_dotenv, find_dotenv\n", + "from pathlib import Path\n", + "from getpass import getpass\n", + "\n", + "print(\"āœ… All imports successful!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b1c8c9e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Chroma telemetry disabled\n" + ] + } + ], + "source": [ + "\n", + "os.environ[\"OPENAI_API_KEY\"] = getpass(\"Enter your OpenAI API key: \") #add openai key\n", + "os.environ[\"CHROMA_TELEMETRY\"] = \"False\" #disable noisy warning\n", + "\n", + "print(\"āœ… Chroma telemetry disabled\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "e6dfa558", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to send telemetry event ClientStartEvent: capture() takes 1 positional argument but 3 were given\n", + "Failed to send telemetry event ClientCreateCollectionEvent: capture() takes 1 positional argument but 3 were given\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ“„ Loaded 1 document(s)\n", + "šŸ“Š Split into 42 chunks\n", + "āœ… State of Union vector store created successfully!\n", + "šŸ“Š Vector store contains 84 documents\n" + ] + } + ], + "source": [ + "# Build State of Union Vector Store\n", + "loader = TextLoader(\"state_of_the_union.txt\")\n", + "documents = loader.load()\n", + "\n", + "print(f\"šŸ“„ Loaded {len(documents)} document(s)\")\n", + "\n", + "# Split into chunks\n", + "text_splitter = CharacterTextSplitter(\n", + " chunk_size=1000,\n", + " chunk_overlap=0\n", + ")\n", + "texts = text_splitter.split_documents(documents)\n", + "\n", + "print(f\"šŸ“Š Split into {len(texts)} chunks\")\n", + "\n", + "# Create embeddings\n", + "embeddings = OpenAIEmbeddings(api_key=os.getenv('OPENAI_API_KEY'))\n", + "\n", + "# Create vector store\n", + "docsearch = Chroma.from_documents(\n", + " texts, \n", + " embeddings, \n", + " collection_name=\"state-of-union\"\n", + ")\n", + "\n", + "print(\"āœ… State of Union vector store created successfully!\")\n", + "print(f\"šŸ“Š Vector store contains {docsearch._collection.count()} documents\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "442046f8", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… RetrievalQA chain created!\n" + ] + } + ], + "source": [ + "# Create RetrievalQA Chain\n", + "\n", + "llm = OpenAI(temperature=0, api_key=os.getenv('OPENAI_API_KEY'))\n", + "\n", + "state_of_union = RetrievalQA.from_chain_type(\n", + " llm=llm, \n", + " chain_type=\"stuff\", \n", + " retriever=docsearch.as_retriever()\n", + ")\n", + "\n", + "print(\"āœ… RetrievalQA chain created!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "22cd02c7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: What did Biden say about Ketanji Brown Jackson?\n", + "āœ… Answer: Biden nominated Ketanji Brown Jackson for the United States Supreme Court.\n" + ] + } + ], + "source": [ + "# Test the Vector Store\n", + "question = \"What did Biden say about Ketanji Brown Jackson?\"\n", + "\n", + "result = state_of_union.run(question)\n", + "print(f\"ā“ Question: {question}\")\n", + "print(f\"āœ… Answer: {result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec8d6e34", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Created a chunk of size 2122, which is longer than the specified 1000\n", + "Created a chunk of size 3187, which is longer than the specified 1000\n", + "Created a chunk of size 1017, which is longer than the specified 1000\n", + "Created a chunk of size 2321, which is longer than the specified 1000\n", + "Failed to send telemetry event ClientStartEvent: capture() takes 1 positional argument but 3 were given\n", + "Failed to send telemetry event ClientCreateCollectionEvent: capture() takes 1 positional argument but 3 were given\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "šŸ“„ Loaded 1 document(s)\n", + "šŸ“Š Split into 23 chunks\n", + "āœ… Ruff vector store created successfully!\n", + "šŸ“Š Vector store contains 69 documents\n" + ] + } + ], + "source": [ + "# Build Ruff Vector Store\n", + "# Load the Ruff FAQ from the web\n", + "loader = WebBaseLoader(\"https://beta.ruff.rs/docs/faq/\")\n", + "docs = loader.load()\n", + "\n", + "print(f\"šŸ“„ Loaded {len(docs)} document(s)\")\n", + "\n", + "# Split into chunks using the same text splitter\n", + "ruff_texts = text_splitter.split_documents(docs)\n", + "\n", + "print(f\"šŸ“Š Split into {len(ruff_texts)} chunks\")\n", + "\n", + "# Create the Ruff vector store\n", + "ruff_db = Chroma.from_documents(\n", + " ruff_texts, \n", + " embeddings, # Reuse the same embeddings\n", + " collection_name=\"ruff\"\n", + ")\n", + "\n", + "print(\"āœ… Ruff vector store created successfully!\")\n", + "print(f\"šŸ“Š Vector store contains {ruff_db._collection.count()} documents\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "a1a2b24f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Ruff RetrievalQA chain created!\n" + ] + } + ], + "source": [ + "# Create Ruff RetrievalQA Chain\n", + "ruff = RetrievalQA.from_chain_type(\n", + " llm=llm, \n", + " chain_type=\"stuff\", \n", + " retriever=ruff_db.as_retriever()\n", + ")\n", + "\n", + "print(\"āœ… Ruff RetrievalQA chain created!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "427e618b", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Failed to send telemetry event CollectionQueryEvent: capture() takes 1 positional argument but 3 were given\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: Why use ruff over flake8?\n", + "āœ… Answer: Ruff offers a larger rule set and the ability to automatically fix lint violations, while also being able to replace other tools such as Black, isort, and pyupgrade. It also supports a wider range of Python versions and does not require the installation of Rust.\n" + ] + } + ], + "source": [ + "# Test Ruff Vector Store\n", + "question = \"Why use ruff over flake8?\"\n", + "\n", + "result = ruff.run(question)\n", + "print(f\"ā“ Question: {question}\")\n", + "print(f\"āœ… Answer: {result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "9b22f987", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "STATE OF UNION QA SYSTEM\n", + "============================================================\n", + "ā“ Q: What did Biden say about Ketanji Brown Jackson?\n", + "āœ… A: Biden nominated Ketanji Brown Jackson for the United States Supreme Court.\n", + "\n", + "============================================================\n", + "RUFF QA SYSTEM\n", + "============================================================\n", + "ā“ Q: What are the benefits of using ruff?\n", + "āœ… A: \n", + "1. Consistent Code Formatting: Ruff's formatter ensures that your code is consistently formatted, making it easier to read and maintain.\n", + "\n", + "2. Comprehensive Linting: Ruff's linter checks for a wide range of potential errors and style violations, helping to catch bugs and improve code quality.\n", + "\n", + "3. Integration with Black: Ruff is compatible with Black, a popular code formatter, making it easy to use both tools together for even more consistent and error-free code.\n", + "\n", + "4. Drop-in Replacement for Flake8: Ruff can be used as a drop-in replacement for Flake8, a popular code analysis tool, making it easy to switch to Ruff without losing any functionality.\n", + "\n", + "5. Customizable Rules: Ruff allows you to customize the linting rules to fit your specific needs and preferences.\n", + "\n", + "6. Python 3 Support: Ruff is designed specifically for Python 3 code, ensuring that it is optimized for the latest version of the language.\n", + "\n", + "7. Popular Plugins Included: Ruff includes popular Flake8 plugins and code quality tools, making it a comprehensive and powerful tool for improving your code.\n" + ] + } + ], + "source": [ + "# Compare Both Vector Stores\n", + "print(\"=\"*60)\n", + "print(\"STATE OF UNION QA SYSTEM\")\n", + "print(\"=\"*60)\n", + "question1 = \"What did Biden say about Ketanji Brown Jackson?\"\n", + "result1 = state_of_union.run(question1)\n", + "print(f\"ā“ Q: {question1}\")\n", + "print(f\"āœ… A: {result1}\")\n", + "\n", + "print(\"\\n\" + \"=\"*60)\n", + "print(\"RUFF QA SYSTEM\")\n", + "print(\"=\"*60)\n", + "question2 = \"What are the benefits of using ruff?\"\n", + "result2 = ruff.run(question2)\n", + "print(f\"ā“ Q: {question2}\")\n", + "print(f\"āœ… A: {result2}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "b4f2d270", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Tools created!\n", + "šŸ“¦ Total tools: 2\n", + " - State of Union QA System\n", + " - Ruff QA System\n" + ] + } + ], + "source": [ + "# Create Tools\n", + "\n", + "tools = [\n", + " Tool(\n", + " name=\"State of Union QA System\",\n", + " func=state_of_union.run,\n", + " description=\"useful for when you need to answer questions about the most recent state of the union address. Input should be a fully formed question.\",\n", + " ),\n", + " Tool(\n", + " name=\"Ruff QA System\",\n", + " func=ruff.run,\n", + " description=\"useful for when you need to answer questions about ruff (a python linter). Input should be a fully formed question.\",\n", + " ),\n", + "]\n", + "\n", + "print(\"āœ… Tools created!\")\n", + "print(f\"šŸ“¦ Total tools: {len(tools)}\")\n", + "for tool in tools:\n", + " print(f\" - {tool.name}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "43163301", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Agent created successfully!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/88/scxzcgb55s1_sms0q1v2bkl80000gn/T/ipykernel_92245/606260814.py:2: LangChainDeprecationWarning: The function `initialize_agent` was deprecated in LangChain 0.1.0 and will be removed in 1.0. Use Use new agent constructor methods like create_react_agent, create_json_agent, create_structured_chat_agent, etc. instead.\n", + " agent = initialize_agent(\n" + ] + } + ], + "source": [ + "# Create the Agent\n", + "agent = initialize_agent(\n", + " tools, \n", + " llm, \n", + " agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, \n", + " verbose=True,\n", + " handle_parsing_errors=True\n", + ")\n", + "\n", + "print(\"āœ… Agent created successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4e6f646", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: What did Biden say about Ketanji Brown Jackson in the state of the union address?\n", + "============================================================\n", + "\u001b[32;1m\u001b[1;3m I should use the State of Union QA System to answer this question.\n", + "Action: State of Union QA System\n", + "Action Input: \"What did Biden say about Ketanji Brown Jackson in the state of the union address?\"\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3m Biden mentioned that he nominated Ketanji Brown Jackson to serve on the United States Supreme Court and praised her as one of the nation's top legal minds who will continue Justice Breyer's legacy of excellence.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I should use the Ruff QA System to check for any errors in the answer.\n", + "Action: Ruff QA System\n", + "Action Input: \"Biden mentioned that he nominated Ketanji Brown Jackson to serve on the United States Supreme Court and praised her as one of the nation's top legal minds who will continue Justice Breyer's legacy of excellence.\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m I don't know.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I should try rephrasing the question to see if I can get a better answer.\n", + "Action: State of Union QA System\n", + "Action Input: \"What was Biden's statement about Ketanji Brown Jackson in the state of the union address?\"\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3m Biden stated that he nominated Circuit Court of Appeals Judge Ketanji Brown Jackson to serve on the United States Supreme Court, and that she is one of the nation's top legal minds who will continue Justice Breyer's legacy of excellence.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m This seems like a more accurate answer.\n", + "Final Answer: Biden nominated Ketanji Brown Jackson to serve on the Supreme Court and praised her as one of the nation's top legal minds who will continue Justice Breyer's legacy of excellence.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: Biden nominated Ketanji Brown Jackson to serve on the Supreme Court and praised her as one of the nation's top legal minds who will continue Justice Breyer's legacy of excellence.\n" + ] + } + ], + "source": [ + "# Test State of Union Question\n", + "question = \"What did Biden say about Ketanji Brown Jackson in the state of the union address?\"\n", + "\n", + "print(f\"ā“ Question: {question}\")\n", + "print(\"=\"*60)\n", + "result = agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01727d97", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: Why use ruff over flake8?\n", + "============================================================\n", + "\u001b[32;1m\u001b[1;3m Ruff is a python linter that has some unique features compared to flake8, so it's worth exploring the differences.\n", + "Action: Ruff QA System\n", + "Action Input: \"What are the unique features of ruff compared to flake8?\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff has a larger rule set, supports automatic fixing of lint violations, and does not support custom or third-party rules. It also has a formatter that is designed to be a drop-in replacement for Black.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m These unique features could be beneficial for certain projects, but it's important to consider the trade-offs.\n", + "Action: State of Union QA System\n", + "Action Input: \"What are the trade-offs of using ruff over flake8?\"\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3m I don't know.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m It seems like the State of Union QA System is not able to answer this question, so let's try asking Ruff QA System again.\n", + "Action: Ruff QA System\n", + "Action Input: \"What are the trade-offs of using ruff over flake8?\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m\n", + "The main trade-off of using Ruff over Flake8 is that Ruff does not support custom lint rules. This means that popular Flake8 plugins must be re-implemented in Rust as part of Ruff itself. Additionally, Ruff may yield slightly different results than Flake8 in some cases, as it uses different rule codes and prefixes. However, Ruff does have the advantage of implementing a larger number of rules (over 900 compared to Flake8's ~409) and being capable of automatically fixing its own lint violations. Ultimately, the decision to use Ruff or Flake8 will depend on the specific needs and preferences of the user.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I now know the final answer.\n", + "Final Answer: The main trade-off of using Ruff over Flake8 is that Ruff does not support custom lint rules, but it has a larger rule set and supports automatic fixing of lint violations. The decision to use Ruff or Flake8 will depend on the specific needs and preferences of the user.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: The main trade-off of using Ruff over Flake8 is that Ruff does not support custom lint rules, but it has a larger rule set and supports automatic fixing of lint violations. The decision to use Ruff or Flake8 will depend on the specific needs and preferences of the user.\n" + ] + } + ], + "source": [ + "# Test Ruff Question\n", + "question = \"Why use ruff over flake8?\"\n", + "\n", + "print(f\"ā“ Question: {question}\")\n", + "print(\"=\"*60)\n", + "result = agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")\n", + "\n", + "# observation: The agent ended up answering a different question (not very far from the original) after the rephrasing attempt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0eeba58e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: What tool does ruff use to run over Jupyter Notebooks? Did the president mention that tool in the state of the union?\n", + "============================================================\n", + "\u001b[32;1m\u001b[1;3m I should check the documentation for ruff and the state of the union address.\n", + "Action: Ruff QA System\n", + "Action Input: What tool does ruff use to run over Jupyter Notebooks?\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff uses nbQA to run over Jupyter Notebooks.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I should now check the state of the union address.\n", + "Action: State of Union QA System\n", + "Action Input: Did the president mention nbQA in the state of the union?\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3m No, the president did not mention nbQA in the state of the union.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m I now know the final answer.\n", + "Final Answer: The final answer is that Ruff uses nbQA to run over Jupyter Notebooks, but the president did not mention it in the state of the union.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: The final answer is that Ruff uses nbQA to run over Jupyter Notebooks, but the president did not mention it in the state of the union.\n" + ] + } + ], + "source": [ + "# Test Multi-Hop Question\n", + "question = \"What tool does ruff use to run over Jupyter Notebooks? Did the president mention that tool in the state of the union?\"\n", + "\n", + "print(f\"ā“ Question: {question}\")\n", + "print(\"=\"*60)\n", + "result = agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")\n", + "#observation: perfect answer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a041f3f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Router tools created with return_direct=True\n" + ] + } + ], + "source": [ + "# Router Tools with direct return\n", + "router_tools = [\n", + " Tool(\n", + " name=\"State of Union QA System\",\n", + " func=state_of_union.run,\n", + " description=\"useful for when you need to answer questions about the most recent state of the union address. Input should be a fully formed question.\",\n", + " return_direct=True,\n", + " ),\n", + " Tool(\n", + " name=\"Ruff QA System\",\n", + " func=ruff.run,\n", + " description=\"useful for when you need to answer questions about ruff (a python linter). Input should be a fully formed question.\",\n", + " return_direct=True,\n", + " ),\n", + "]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "c31008b2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "āœ… Router agent created!\n" + ] + } + ], + "source": [ + "# Router Agent\n", + "router_agent = initialize_agent(\n", + " router_tools, \n", + " llm, \n", + " agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, \n", + " verbose=True,\n", + " handle_parsing_errors=True\n", + ")\n", + "\n", + "print(\"āœ… Router agent created!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "ea78cf55", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: What did Biden say about Ketanji Brown Jackson?\n", + "============================================================\n", + "\u001b[32;1m\u001b[1;3m It's important to know the context of the question and what information is being sought.\n", + "Action: State of Union QA System\n", + "Action Input: \"What did Biden say about Ketanji Brown Jackson?\"\u001b[0m\n", + "Observation: \u001b[36;1m\u001b[1;3m Biden nominated Ketanji Brown Jackson for the United States Supreme Court.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: Biden nominated Ketanji Brown Jackson for the United States Supreme Court.\n" + ] + } + ], + "source": [ + "# Test Router Agent with State of Union\n", + "question = \"What did Biden say about Ketanji Brown Jackson?\"\n", + "\n", + "print(f\"ā“ Question: {question}\")\n", + "print(\"=\"*60)\n", + "result = router_agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96e0769e", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "======================================================================\n", + "COMPARISON: NORMAL AGENT vs ROUTER AGENT\n", + "======================================================================\n", + "\n", + "ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n", + "1ļøāƒ£ NORMAL AGENT (with extra reasoning)\n", + "ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n", + "\u001b[32;1m\u001b[1;3m Ruff is a python linter that has some unique features compared to flake8, so it may be useful in certain situations.\n", + "Action: Ruff QA System\n", + "Action Input: \"Why use ruff over flake8?\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff offers a larger rule set and the ability to automatically fix lint violations, while also being able to replace other tools such as Black, isort, and pyupgrade. It also supports Python 3.13 and does not require the installation of Rust.\u001b[0m\n", + "Thought:\u001b[32;1m\u001b[1;3m This information is helpful, but I should also consider the benefits of using flake8.\n", + "Action: Ruff QA System\n", + "Action Input: \"What are the benefits of using flake8?\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Flake8 is a popular Python linter that helps identify and fix coding errors and style violations in Python code. It can help improve code quality, readability, and maintainability by enforcing coding standards and best practices. It also helps catch potential bugs and errors before they cause issues in production. Additionally, flake8 can be easily integrated into development workflows and can save time and effort in code reviews.\u001b[0m\n", + "Thought:" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3m I now have a better understanding of the differences between ruff and flake8.\n", + "Final Answer: Depending on the specific needs and preferences of a project, either ruff or flake8 may be a better choice for linting Python code. It is important to consider the features and benefits of each tool before making a decision.\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "Answer: Depending on the specific needs and preferences of a project, either ruff or flake8 may be a better choice for linting Python code. It is important to consider the features and benefits of each tool before making a decision.\n", + "\n", + "ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n", + "2ļøāƒ£ ROUTER AGENT (returns directly)\n", + "ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€ā”€\n", + "\u001b[32;1m\u001b[1;3m Ruff is a python linter that has some unique features compared to flake8, so it may be useful in certain situations.\n", + "Action: Ruff QA System\n", + "Action Input: \"Why use ruff over flake8?\"\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff offers a larger rule set and the ability to automatically fix lint violations, while also being able to replace other tools such as Black, isort, and pyupgrade. It also supports a wider range of Python versions and does not require the installation of Rust.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "Answer: Ruff offers a larger rule set and the ability to automatically fix lint violations, while also being able to replace other tools such as Black, isort, and pyupgrade. It also supports a wider range of Python versions and does not require the installation of Rust.\n" + ] + } + ], + "source": [ + "# Compare Agent Behavior\n", + "print(\"=\"*70)\n", + "print(\"COMPARISON: NORMAL AGENT vs ROUTER AGENT\")\n", + "print(\"=\"*70)\n", + "\n", + "question = \"Why use ruff over flake8?\"\n", + "\n", + "print(\"\\n\" + \"ā”€\"*70)\n", + "print(\"1ļøāƒ£ NORMAL AGENT (with extra reasoning)\")\n", + "print(\"ā”€\"*70)\n", + "normal_result = agent.invoke({\"input\": question})\n", + "print(f\"Answer: {normal_result['output']}\")\n", + "\n", + "print(\"\\n\" + \"ā”€\"*70)\n", + "print(\"2ļøāƒ£ ROUTER AGENT (returns directly)\")\n", + "print(\"ā”€\"*70)\n", + "router_result = router_agent.invoke({\"input\": question})\n", + "print(f\"Answer: {router_result['output']}\")\n", + "\n", + "#observation: Same end result with one question. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "65276bf8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ā“ Question: What tool does ruff use to run over Jupyter Notebooks? Did the president mention that tool in the state of the union?\n", + "============================================================\n", + "\u001b[32;1m\u001b[1;3m I should check the documentation for ruff and the state of the union address.\n", + "Action: Ruff QA System\n", + "Action Input: What tool does ruff use to run over Jupyter Notebooks?\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff uses nbQA to run over Jupyter Notebooks.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: Ruff uses nbQA to run over Jupyter Notebooks.\n" + ] + } + ], + "source": [ + "# Test Router Agent with Multi-Hop\n", + "question = \"What tool does ruff use to run over Jupyter Notebooks? Did the president mention that tool in the state of the union?\"\n", + "\n", + "print(f\"ā“ Question: {question}\")\n", + "print(\"=\"*60)\n", + "result = router_agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")\n", + "# observation: agent completely disregarded the second part of the question because of the use of return-direct" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0bfae45", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32;1m\u001b[1;3mThought: Do I need to use a tool? Yes\n", + "Action: Ruff QA System\n", + "Action Input: What tool does ruff use to run over Jupyter Notebooks?\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff uses nbQA to run over Jupyter Notebooks.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… Answer: Ruff uses nbQA to run over Jupyter Notebooks.\n" + ] + } + ], + "source": [ + "# Try a different agent type with the same router_tools\n", + "\n", + "memory = ConversationBufferMemory(\n", + " memory_key=\"chat_history\",\n", + " return_messages=True\n", + ")\n", + "conversational_agent = initialize_agent(\n", + " router_tools,\n", + " llm,\n", + " agent=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,\n", + " verbose=True,\n", + " handle_parsing_errors=True,\n", + " memory=memory\n", + ")\n", + "\n", + "# Test\n", + "question = \"What tool does ruff use to run over Jupyter Notebooks? Did the president mention that tool in the state of the union?\"\n", + "result = conversational_agent.invoke({\"input\": question})\n", + "print(f\"āœ… Answer: {result['output']}\")\n", + "\n", + "# Observation: Changing the agent did not help bypass the return_direct " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ffb68027", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "============================================================\n", + "ROUTER AGENT WITH MEMORY\n", + "============================================================\n", + "\n", + "ā“ Q1: What is ruff?\n", + "\u001b[32;1m\u001b[1;3m\n", + "Thought: Do I need to use a tool? Yes\n", + "Action: Ruff QA System\n", + "Action Input: What is ruff?\u001b[0m" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error in StdOutCallbackHandler.on_chain_start callback: AttributeError(\"'NoneType' object has no attribute 'get'\")\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff is a Python linter and formatter that can replace other tools such as Black, isort, yesqa, eradicate, and most of the rules implemented in pyupgrade. It supports Python versions 3.7 and above and can be installed without needing to install Rust.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… A1: Ruff is a Python linter and formatter that can replace other tools such as Black, isort, yesqa, eradicate, and most of the rules implemented in pyupgrade. It supports Python versions 3.7 and above and can be installed without needing to install Rust.\n", + "\n", + "ā“ Q2: Why is it better than flake8?\n", + "\u001b[32;1m\u001b[1;3m\n", + "Thought: Do I need to use a tool? Yes\n", + "Action: Ruff QA System\n", + "Action Input: Why is it better than flake8?\u001b[0m\n", + "Observation: \u001b[33;1m\u001b[1;3m Ruff is better than flake8 because it implements over 900 rules, compared to flake8's ~80 rules. Additionally, Ruff is capable of automatically fixing its own lint violations, while flake8 does not have this capability. Ruff also has better compatibility with other tools, such as type checkers, and can be used to replace flake8 when used with certain plugins.\u001b[0m\n", + "\u001b[32;1m\u001b[1;3m\u001b[0m\n", + "\n", + "\u001b[1m> Finished chain.\u001b[0m\n", + "āœ… A2: Ruff is better than flake8 because it implements over 900 rules, compared to flake8's ~80 rules. Additionally, Ruff is capable of automatically fixing its own lint violations, while flake8 does not have this capability. Ruff also has better compatibility with other tools, such as type checkers, and can be used to replace flake8 when used with certain plugins.\n" + ] + } + ], + "source": [ + "# Bonus: Router Agent with Memory\n", + "\n", + "router_agent_with_memory = initialize_agent(\n", + " router_tools,\n", + " llm,\n", + " agent=AgentType.CONVERSATIONAL_REACT_DESCRIPTION,\n", + " memory=memory,\n", + " verbose=True\n", + ")\n", + "\n", + "# Test with conversation\n", + "print(\"=\"*60)\n", + "print(\"ROUTER AGENT WITH MEMORY\")\n", + "print(\"=\"*60)\n", + "\n", + "q1 = \"What is ruff?\"\n", + "print(f\"\\nā“ Q1: {q1}\")\n", + "r1 = router_agent_with_memory.invoke({\"input\": q1})\n", + "print(f\"āœ… A1: {r1['output']}\")\n", + "\n", + "# Follow-up question (uses memory)\n", + "q2 = \"Why is it better than flake8?\"\n", + "print(f\"\\nā“ Q2: {q2}\")\n", + "r2 = router_agent_with_memory.invoke({\"input\": q2})\n", + "print(f\"āœ… A2: {r2['output']}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv (3.9.6)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}