Core fixes. Provide global variables. Update runner. Upgrade module-package system. (#14)

* Clean init all the modules with the __init__.py file

* Remove unused gitkeeps

* Add base inits for script packages

* Update main file

* Set __file__ variable at the module level

* Add inits for the modules

* Modify all the modules

* Delete one more .gitkeep

* Add WIP test for user_greeting

* Update test module for user_greeting

* Fix module runner a bit

* Add relative imports to the modules

* Add run tests script

* Update README.md

* Fix module-package runner

* Update requirements

* Format everything with Black

* Handle module and execution errors with runner

Author: manmolecular

README.md

@@ -6,11 +6,79 @@
 virtualenv -p python3 venv
     (or python3 -m venv venv)
 pip3 install -r requirements.txt
+```
+
+## Running as a framework
+_First of all: provide some arguments in the `main.py` file to collect information based on your data (WIP now, will be improved later)_  
+
+To run the framework:
+```bash
 python3 main.py
 ```
+To run the tests:
+```bash
+chmod +x run_tests.sh
+./run_tests.sh
+```
+_or you can run them like this, of course:_
+```bash
+python3 -m unittest discover -v
+```
+## Running as a separated module
+Basic:
+```python3
+python3 -m src.scripts.<category>.<name> any_arguments_here
+```
+Example command:
+```bash
+python3 -m src.scripts.other.user_greeting JohnDoe
+```
+Example output:
+```
+{'message': "Successfully finished! (args: (), kwargs: {'username': "
+            "'johndoe'})",
+ 'result': 'Hello, JohnDoe!',
+ 'status': 'success'}
+
+```
 
 ## Create your own script
 Use the following structure:  
+1. Create your own module directory in the following way:
+```
+/src/scripts/<choose_your_category_here>/<your_script_name>/<script_files>
+```
+2. Provide the following structure of your script directory:
+```
+your_script_name
+├── __init__.py     - use this module to set the default parent directory (you can copy this file from any other script)
+├── __main__.py     - use this module to provide some basic interface to use your script as a module (the same as if __name__ == "__main__")
+├── module.py       - use this module to describe the basic logic of your module (you can import it in the __main__.py to provide interface)
+└── test_module.py  - use this module for unittest tests
+```
+3. Create the `__init__.py` file. An example of the `__init__.py` boilerplate structure can be seen below:
+```python3
+import sys
+from pathlib import Path
+
+__root_dir = Path(__file__).parents[4]
+sys.path.append(str(__root_dir))
+
+```
+4. Create the `__main__.py` file. An example of the `__main__.py` boilerplate structure can be seen below:
+```python3
+#!/usr/bin/env python3
+
+from pprint import pprint
+from sys import argv
+
+from src.core.utils.module import run_module
+from .module import Runner
+
+result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe")
+pprint(result)
+```
+5. Create the module itself. An example of the basic `module.py` file can be seen below:
 ```python3
 #!/usr/bin/env python3
 
@@ -60,3 +128,4 @@ class Runner(OsintRunner):
         argument = kwargs.get("my_argument", "Arguments were not provided!")
         return ScriptResponse.success(message=f"Script finished with argument {argument}")
 ```
+6. For `test_module.py` you can use any required tests (as you wish). A test case for your module is required to keep the project clean.

main.py

@@ -13,26 +13,33 @@
 from src.core.case.osint import OsintCase
 from src.core.case.recon import ReconCase
 from src.core.case.base import BaseCase
+from src.core.utils.log import Logger
+
+
+logger = Logger.get_logger(name="osint-framework")
+
+
+def run_case(case_class: type, *args, **kwargs):
+    """
+    Define and smoke run the BaseCase
+    :param case_class: original class of the case
+    :param args: some args
+    :param kwargs: some kwargs
+    :return: result of the execution
+    """
+    logger.info(f"start {case_class.__name__} case processing")
+    case = case_class()
+    case.process(*args, **kwargs)
+    return case.get_results()
 
 
 if __name__ == "__main__":
-    # Will return 2 results from "other" category scripts
-    other_case = BaseCase()
-    other_case.process(
-        username="johndoe", email="[email protected]", fullname="John Doe",
-    )
-    other_case_results = other_case.get_results()
-
-    # Will return 1 result from "recon" category scripts
-    recon_case = ReconCase()
-    recon_case.process(url="https://facebook.com")
-    recon_case_results = recon_case.get_results()
-
-    # Will return nothing (no scripts for now, sorry!)
-    osint_case = OsintCase()
-    osint_case.process(username="any_value_here")
-    osint_case_results = osint_case.get_results()
-
-    # Print out all the results
-    for result in other_case_results, recon_case_results, osint_case_results:
-        pprint(result)
+    # fmt: off
+    base_case = run_case(case_class=BaseCase, username="johndoe", email="[email protected]", fullname="John Doe")
+    osint_case = run_case(case_class=OsintCase, username="johndoe", email="[email protected]", fullname="John Doe")
+    recon_case = run_case(case_class=ReconCase, url="https://facebook.com")
+
+    pprint(base_case)
+    pprint(osint_case)
+    pprint(recon_case)
+    # fmt: on

requirements.txt

@@ -1,5 +1,18 @@
+aiodns==2.0.0
+aiohttp==3.6.2
+aiosmtpd==1.2
+async-timeout==3.0.1
+atpublic==1.0
+attrs==19.3.0
 certifi==2020.6.20
+cffi==1.14.0
 chardet==3.0.4
 idna==2.10
+mmh3==2.5.1
+multidict==4.7.6
+pycares==3.1.1
+pycparser==2.20
 requests==2.24.0
 urllib3==1.25.9
+verify-email==2.4.1
+yarl==1.4.2

run_tests.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+python3 -m unittest discover -v

src/core/exceptions/.gitkeep src/__init__.py

@@ -32,14 +32,14 @@ def __init__(self):
         self.scripts = {}
         self.results = {}
 
+    @staticmethod
     def exec_script(
-        self,
         path: str or Path,
         script_class: str = "Runner",
         function: str = "run",
         args: list or None = None,
         kwargs: dict or None = None,
-    ) -> ScriptResponse:
+    ) -> ScriptResponse or dict:
         """
         Load and exec python script
         :param path: name of the script to load
@@ -55,13 +55,34 @@ def exec_script(
             kwargs = {}
         loader = SourceFileLoader(fullname=script_class, path=str(path))
         module = ModuleType(name=loader.name)
-        loader.exec_module(module)
-        class_instance = getattr(module, script_class)(logger=path.parent.stem)
+        result = {"script": Path(path).parent.stem}
+
+        # Check if don't forget to install all the dependencies, and that module can
+        # be successfully loaded.
+
+        # fmt: off
+        try:
+            loader.exec_module(module)
+        except Exception as unexp_err:
+            result.update(ScriptResponse.error(message=f"Unexpected module error: {str(unexp_err)}"))
+            return result
+        # fmt: on
+
+        # Module successfully loaded. We can set some module-scope variables that
+        # we missed.
+        module.__file__ = path
+
+        # Execute the runner and check if something goes wrong.
+
+        # fmt: off
         try:
-            result = getattr(class_instance, function)(*args, **kwargs)
+            class_instance = getattr(module, script_class)(logger=path.parent.stem)
+            result.update(getattr(class_instance, function)(*args, **kwargs))
         except Exception as unexp_err:
-            result = ScriptResponse.error(message=str(unexp_err))
-        result.update({"script": Path(path).parent.stem})
+            result.update(ScriptResponse.error(message=f"Unexpected execution error: {str(unexp_err)}"))
+        # fmt: on
+
+        # In any possible case, return result from the module or ScriptResponse.error + script name
         return result
 
     def get_scripts(self) -> dict:
@@ -76,7 +97,7 @@ def get_scripts(self) -> dict:
             ScriptRunnerPaths.CONVERT,
         ]:
             self.scripts.update({directory.stem: list()})
-            for file in directory.glob("*/__main__.py"):
+            for file in directory.glob("*/module.py"):
                 self.scripts[directory.stem].append(file)
         return self.scripts
 

src/core/handlers/.gitkeep src/core/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+
+def run_module(
+    runner: type, args: list, arg_name: str, arg_default: str or None = None
+) -> dict:
+    """
+    Use module as a 'python3 -m ...' module
+    :param runner: runner object
+    :param args: list of args from CLI
+    :param arg_name: name of arg to use
+    :param arg_default: default arg value
+    :return: results
+    """
+    runner = runner()
+    return runner.run(**{arg_name: args[1] if len(args) >= 2 else arg_default})

src/scripts/convert/.gitkeep src/core/base/__init__.py

@@ -0,0 +1,5 @@
+import sys
+from pathlib import Path
+
+__root_dir = Path(__file__).parents[4]
+sys.path.append(str(__root_dir))

src/scripts/osint/.gitkeep src/core/case/__init__.py

@@ -1,98 +1,10 @@
 #!/usr/bin/env python3
 
-import asyncio
-from pathlib import Path
+from pprint import pprint
+from sys import argv
 
-import aiohttp
-import requests
+from src.core.utils.module import run_module
+from .module import Runner
 
-from src.core.base.osint import OsintRunner
-from src.core.utils.response import ScriptResponse
-
-
-class Defaults:
-    NETWORKS_LIST = "social_networks.txt"
-
-
-class Networks:
-    def __init__(self):
-        with open(Path("./src/scripts/osint/check_nickname/data/social_networks.txt")) as file:
-            self.net = file.read().splitlines()
-
-
-def check_nickname_sync(nickname: str) -> list:
-    """
-    checks nicknames from social networks(sync)
-    :param nickname: just nickname :)
-    :return: list with links to user from social
-     networks which have this nickname
-    """
-    ans = []
-    social = Networks().net
-    for site in social:
-        try:
-            url = "https://{site}{nickname}".format(site=site, nickname=nickname)
-            response = requests.get(url)
-            if response.status_code == 200:
-                ans.append(url)
-        except:
-            pass
-    return ans
-
-
-async def check_nickname_async(nickname: str, social) -> list:
-    """
-    checks nicknames from social networks(async)
-    :param nickname: just nickname :)
-    :param social: social
-    :return: list with links to user from social
-    networks which have this nickname
-    """
-    ans = []
-    async with aiohttp.ClientSession() as session:
-        while not social.empty():
-            url = await social.get()
-            try:
-                async with session.get(url) as response:
-                    if response.status == 200:
-                        ans.append(url)
-            except:
-                pass
-    return ans
-
-
-class Runner(OsintRunner):
-    def __init__(self, logger: str = __name__):
-        super().__init__(logger=logger)
-
-    @staticmethod
-    async def __run(*args, **kwargs):
-        try:
-            username = kwargs.get("username")
-            social = asyncio.Queue()
-            for site in Networks().net:
-                await social.put(
-                    "https://{site}{username}".format(site=site, username=username)
-                )
-            temp_result = await asyncio.gather(
-                *[
-                    asyncio.create_task(check_nickname_async(username, social))
-                    for _ in range(10)
-                ]
-            )
-            result = {username: []}
-            for sub_massive in temp_result:
-                for site in sub_massive:
-                    result[username].append(site)
-            return ScriptResponse.success(
-                result=result,
-                message="Found {count} user accounts".format(
-                    count=len(result[username])
-                ),
-            )
-        except Exception as err:
-            return ScriptResponse.error(message=str(err))
-
-    def run(self, *args, **kwargs):
-        username = kwargs.get("username")
-        return asyncio.run(self.__run(username=username))
+result = run_module(Runner, args=argv, arg_name="username", arg_default="johndoe")
+pprint(result)