Merge pull request Significant-Gravitas#2192 from merwanehamadi/featu…

…re/change-flake-config

Align flake, black and isort to pipelines and precommit hooks

.flake8

@@ -1,12 +1,12 @@
 [flake8]
 max-line-length = 88
-extend-ignore = E203
+select = "E303, W293, W291, W292, E305, E231, E302"
 exclude =
     .tox,
     __pycache__,
     *.pyc,
     .env
-    venv/*
-    .venv/*
-    reports/*
-    dist/*
+    venv*/*,
+    .venv/*,
+    reports/*,
+    dist/*,

.github/workflows/ci.yml

@@ -32,7 +32,15 @@ jobs:
 
     - name: Lint with flake8
       continue-on-error: false
-      run: flake8 autogpt/ tests/ --select E303,W293,W291,W292,E305,E231,E302
+      run: flake8
+
+    - name: Check black formatting
+      continue-on-error: false
+      run: black . --check
+
+    - name: Check isort formatting
+      continue-on-error: false
+      run: isort . --check
 
     - name: Run unittest tests with coverage
       run: |

.pre-commit-config.yaml

@@ -1,39 +1,32 @@
 repos:
-  - repo: https://github.com/sourcery-ai/sourcery
-    rev: v1.1.0 # Get the latest tag from https://github.com/sourcery-ai/sourcery/tags
-    hooks:
-      - id: sourcery
-
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v0.9.2
     hooks:
       - id: check-added-large-files
-        args: [ '--maxkb=500' ]
+        args: ['--maxkb=500']
       - id: check-byte-order-marker
       - id: check-case-conflict
       - id: check-merge-conflict
       - id: check-symlinks
       - id: debug-statements
-
-  - repo: local
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
     hooks:
       - id: isort
-        name: isort-local
-        entry: isort
-        language: python
-        types: [ python ]
-        exclude: .+/(dist|.venv|venv|build)/.+
-        pass_filenames: true
+        language_version: python3.10
+
+  - repo: https://github.com/psf/black
+    rev: 23.3.0
+    hooks:
       - id: black
-        name: black-local
-        entry: black
-        language: python
-        types: [ python ]
-        exclude: .+/(dist|.venv|venv|build)/.+
-        pass_filenames: true
+        language_version: python3.10
+
+  - repo: local
+    hooks:
       - id: pytest-check
         name: pytest-check
         entry: pytest --cov=autogpt --without-integration --without-slow-integration
         language: system
         pass_filenames: false
-        always_run: true
+        always_run: true

autogpt/__main__.py

@@ -1,12 +1,15 @@
 """Main script for the autogpt package."""
 import logging
+
 from colorama import Fore
+
 from autogpt.agent.agent import Agent
 from autogpt.args import parse_arguments
 from autogpt.config import Config, check_openai_api_key
 from autogpt.logs import logger
 from autogpt.memory import get_memory
 from autogpt.prompt import construct_prompt
+
 # Load environment variables from .env file
 
 

autogpt/agent/agent.py

@@ -1,6 +1,6 @@
 from colorama import Fore, Style
-from autogpt.app import execute_command, get_command
 
+from autogpt.app import execute_command, get_command
 from autogpt.chat import chat_with_ai, create_chat_message
 from autogpt.config import Config
 from autogpt.json_fixes.master_json_fix_method import fix_json_using_multiple_techniques
@@ -84,7 +84,7 @@ def start_interaction_loop(self):
 
             # Print Assistant thoughts
             if assistant_reply_json != {}:
-                validate_json(assistant_reply_json, 'llm_response_format_1')
+                validate_json(assistant_reply_json, "llm_response_format_1")
                 # Get command name and arguments
                 try:
                     print_assistant_thoughts(self.ai_name, assistant_reply_json)

autogpt/agent/agent_manager.py

@@ -1,8 +1,10 @@
 """Agent manager for managing GPT agents"""
 from __future__ import annotations
+
 from typing import Union
-from autogpt.llm_utils import create_chat_completion
+
 from autogpt.config.config import Singleton
+from autogpt.llm_utils import create_chat_completion
 
 
 class AgentManager(metaclass=Singleton):

autogpt/app.py

@@ -1,15 +1,10 @@
 """ Command and Control """
 import json
-from typing import List, NoReturn, Union, Dict
+from typing import Dict, List, NoReturn, Union
+
 from autogpt.agent.agent_manager import AgentManager
-from autogpt.commands.evaluate_code import evaluate_code
-from autogpt.commands.google_search import google_official_search, google_search
-from autogpt.commands.improve_code import improve_code
-from autogpt.commands.write_tests import write_tests
-from autogpt.config import Config
-from autogpt.commands.image_gen import generate_image
 from autogpt.commands.audio_text import read_audio_from_file
-from autogpt.commands.web_requests import scrape_links, scrape_text
+from autogpt.commands.evaluate_code import evaluate_code
 from autogpt.commands.execute_code import (
     execute_python_file,
     execute_shell,
@@ -18,19 +13,24 @@
 from autogpt.commands.file_operations import (
     append_to_file,
     delete_file,
+    download_file,
     read_file,
     search_files,
     write_to_file,
-    download_file
 )
+from autogpt.commands.git_operations import clone_repository
+from autogpt.commands.google_search import google_official_search, google_search
+from autogpt.commands.image_gen import generate_image
+from autogpt.commands.improve_code import improve_code
+from autogpt.commands.twitter import send_tweet
+from autogpt.commands.web_requests import scrape_links, scrape_text
+from autogpt.commands.web_selenium import browse_website
+from autogpt.commands.write_tests import write_tests
+from autogpt.config import Config
 from autogpt.json_fixes.parsing import fix_and_parse_json
 from autogpt.memory import get_memory
 from autogpt.processing.text import summarize_text
 from autogpt.speech import say_text
-from autogpt.commands.web_selenium import browse_website
-from autogpt.commands.git_operations import clone_repository
-from autogpt.commands.twitter import send_tweet
-
 
 CFG = Config()
 AGENT_MANAGER = AgentManager()
@@ -133,11 +133,14 @@ def execute_command(command_name: str, arguments):
 
             # google_result can be a list or a string depending on the search results
             if isinstance(google_result, list):
-                safe_message = [google_result_single.encode('utf-8', 'ignore') for google_result_single in google_result]
+                safe_message = [
+                    google_result_single.encode("utf-8", "ignore")
+                    for google_result_single in google_result
+                ]
             else:
-                safe_message = google_result.encode('utf-8', 'ignore')
+                safe_message = google_result.encode("utf-8", "ignore")
 
-            return safe_message.decode('utf-8')
+            return safe_message.decode("utf-8")
         elif command_name == "memory_add":
             memory = get_memory(CFG)
             return memory.add(arguments["string"])

autogpt/args.py

@@ -1,7 +1,8 @@
 """This module contains the argument parsing logic for the script."""
 import argparse
 
-from colorama import Fore, Back, Style
+from colorama import Back, Fore, Style
+
 from autogpt import utils
 from autogpt.config import Config
 from autogpt.logs import logger
@@ -64,10 +65,10 @@ def parse_arguments() -> None:
         " skip the re-prompt.",
     )
     parser.add_argument(
-        '--allow-downloads',
-        action='store_true',
-        dest='allow_downloads',
-        help='Dangerous: Allows Auto-GPT to download files natively.'
+        "--allow-downloads",
+        action="store_true",
+        dest="allow_downloads",
+        help="Dangerous: Allows Auto-GPT to download files natively.",
     )
     args = parser.parse_args()
 
@@ -141,10 +142,17 @@ def parse_arguments() -> None:
 
     if args.allow_downloads:
         logger.typewriter_log("Native Downloading:", Fore.GREEN, "ENABLED")
-        logger.typewriter_log("WARNING: ", Fore.YELLOW,
-                              f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} " +
-                              "It is recommended that you monitor any files it downloads carefully.")
-        logger.typewriter_log("WARNING: ", Fore.YELLOW, f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}")
+        logger.typewriter_log(
+            "WARNING: ",
+            Fore.YELLOW,
+            f"{Back.LIGHTYELLOW_EX}Auto-GPT will now be able to download and save files to your machine.{Back.RESET} "
+            + "It is recommended that you monitor any files it downloads carefully.",
+        )
+        logger.typewriter_log(
+            "WARNING: ",
+            Fore.YELLOW,
+            f"{Back.RED + Style.BRIGHT}ALWAYS REMEMBER TO NEVER OPEN FILES YOU AREN'T SURE OF!{Style.RESET_ALL}",
+        )
         CFG.allow_downloads = True
 
     if args.browser_name:

autogpt/commands/audio_text.py

@@ -1,6 +1,7 @@
-import requests
 import json
 
+import requests
+
 from autogpt.config import Config
 from autogpt.workspace import path_in_workspace
 

autogpt/commands/execute_code.py

@@ -5,7 +5,7 @@
 import docker
 from docker.errors import ImageNotFound
 
-from autogpt.workspace import path_in_workspace, WORKSPACE_PATH
+from autogpt.workspace import WORKSPACE_PATH, path_in_workspace
 
 
 def execute_python_file(file: str) -> str:

autogpt/commands/file_operations.py

@@ -5,14 +5,14 @@
 import os.path
 from pathlib import Path
 from typing import Generator, List
+
 import requests
-from requests.adapters import HTTPAdapter
-from requests.adapters import Retry
-from colorama import Fore, Back
+from colorama import Back, Fore
+from requests.adapters import HTTPAdapter, Retry
+
 from autogpt.spinner import Spinner
 from autogpt.utils import readable_file_size
-from autogpt.workspace import path_in_workspace, WORKSPACE_PATH
-
+from autogpt.workspace import WORKSPACE_PATH, path_in_workspace
 
 LOG_FILE = "file_logger.txt"
 LOG_FILE_PATH = WORKSPACE_PATH / LOG_FILE
@@ -47,7 +47,7 @@ def log_operation(operation: str, filename: str) -> None:
         with open(LOG_FILE_PATH, "w", encoding="utf-8") as f:
             f.write("File Operation Logger ")
 
-    append_to_file(LOG_FILE, log_entry, shouldLog = False)
+    append_to_file(LOG_FILE, log_entry, shouldLog=False)
 
 
 def split_file(
@@ -241,23 +241,23 @@ def download_file(url, filename):
             session = requests.Session()
             retry = Retry(total=3, backoff_factor=1, status_forcelist=[502, 503, 504])
             adapter = HTTPAdapter(max_retries=retry)
-            session.mount('http://', adapter)
-            session.mount('https://', adapter)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
 
             total_size = 0
             downloaded_size = 0
 
             with session.get(url, allow_redirects=True, stream=True) as r:
                 r.raise_for_status()
-                total_size = int(r.headers.get('Content-Length', 0))
+                total_size = int(r.headers.get("Content-Length", 0))
                 downloaded_size = 0
 
-                with open(safe_filename, 'wb') as f:
+                with open(safe_filename, "wb") as f:
                     for chunk in r.iter_content(chunk_size=8192):
                         f.write(chunk)
                         downloaded_size += len(chunk)
 
-                         # Update the progress message
+                        # Update the progress message
                         progress = f"{readable_file_size(downloaded_size)} / {readable_file_size(total_size)}"
                         spinner.update_message(f"{message} {progress}")
 

autogpt/commands/git_operations.py

@@ -1,5 +1,6 @@
 """Git operations for autogpt"""
 import git
+
 from autogpt.config import Config
 from autogpt.workspace import path_in_workspace
 

autogpt/commands/image_gen.py

@@ -7,6 +7,7 @@
 import openai
 import requests
 from PIL import Image
+
 from autogpt.config import Config
 from autogpt.workspace import path_in_workspace
 

autogpt/commands/twitter.py

@@ -1,5 +1,6 @@
-import tweepy
 import os
+
+import tweepy
 from dotenv import load_dotenv
 
 load_dotenv()

autogpt/commands/web_playwright.py

@@ -8,6 +8,7 @@
         "Playwright not installed. Please install it with 'pip install playwright' to use."
     )
 from bs4 import BeautifulSoup
+
 from autogpt.processing.html import extract_hyperlinks, format_hyperlinks
 
 

autogpt/commands/web_requests.py

@@ -4,9 +4,9 @@
 from urllib.parse import urljoin, urlparse
 
 import requests
-from requests.compat import urljoin
-from requests import Response
 from bs4 import BeautifulSoup
+from requests import Response
+from requests.compat import urljoin
 
 from autogpt.config import Config
 from autogpt.memory import get_memory
@@ -79,7 +79,7 @@ def check_local_file_access(url: str) -> bool:
         "http://0000",
         "http://0000/",
         "https://0000",
-        "https://0000/"
+        "https://0000/",
     ]
     return any(url.startswith(prefix) for prefix in local_prefixes)