WIP: Add Initial Version (alpha-beta-gamma) (#1)

* Add initial WIP project version

* Update src code

* Update documentation

* Add requirements

Author: manmolecular

.gitignore

@@ -1,3 +1,6 @@
+# Custom
+.idea/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,2 +1,62 @@
 # osint-framework
-:eyes: WIP: All-in-one OSINT Tools
+:eyes: All-in-one OSINT/RECON Swiss Knife
+
+## Installing
+```bash
+virtualenv -p python3 venv
+    (or python3 -m venv venv)
+pip3 install -r requirements.txt
+python3 main.py
+```
+
+## Create your own script
+Use the following structure:  
+```python3
+#!/usr/bin/env python3
+
+# Import any required runner
+# 1. OsintRunner - for OSINT scripts
+# 2. ReconRunner - for RECON scripts
+# 3. BaseRunner - for out-of-scope scripts ("other")
+from src.core.base.osint import OsintRunner, BaseRunner, ReconRunner, PossibleKeys
+
+# Import 'ScriptResponse' to return good responses from the module, like
+# 1. ScriptResponse.success - if everything is good
+# 2. ScriptResponse.error - if everything is bad
+from src.core.utils.response import ScriptResponse
+
+# Validate your named arguments. For example, this validator
+# will raise 'KeyError' if you will try to put 'hostname' argument
+# into the 'OsintRunner' runner, and so on
+from src.core.utils.validators import validate_kwargs
+
+# You can use OsintRunner, ReconRunner or BaseRunner as the base class
+class Runner(OsintRunner):
+    """
+    Basic script example
+    """
+
+    def __init__(self, logger: str = __name__):
+        """
+        Re-init base class instance with this function.
+        Simply put, you need to provide proper logger name
+        to the parent class, so please, save this structure for
+        the init function.
+        :param logger: logger to use (name of _this_ runner by default)
+        """
+        super(Runner, self).__init__(logger)
+
+    @validate_kwargs(PossibleKeys.KEYS)
+    def run(self, *args, **kwargs) -> ScriptResponse.success or ScriptResponse.error:
+        """
+        The main '.run()' function to run your script. 
+        Note: this function is always synchronous, without any
+        async/await init. You can use 'asyncio.run(...)' here,
+        but don't put any 'async' before function definition
+        :param args: args that you provide (not used for now)
+        :param kwargs: kwargs that you provide (required to run something!)
+        :return: ScriptResponse message (error or success)
+        """
+        argument = kwargs.get("my_argument", "Arguments were not provided!")
+        return ScriptResponse.success(message=f"Script finished with argument {argument}")
+```

main.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+"""
+Main runner.
+
+Please, for all the searches, use the following classes:
+'BaseCase', 'OsintCase', 'ReconCase'
+"""
+
+
+from pprint import pprint
+
+from src.core.case.osint import OsintCase
+from src.core.case.recon import ReconCase
+from src.core.case.base import BaseCase
+
+
+if __name__ == "__main__":
+    # Will return 2 results from "other" category scripts
+    other_case = BaseCase()
+    other_case.process(
+        username="johndoe", email="[email protected]", fullname="John Doe",
+    )
+    other_case_results = other_case.get_results()
+
+    # Will return 1 result from "recon" category scripts
+    recon_case = ReconCase()
+    recon_case.process(url="https://facebook.com")
+    recon_case_results = recon_case.get_results()
+
+    # Will return nothing (no scripts for now, sorry!)
+    osint_case = OsintCase()
+    osint_case.process(username="any_value_here")
+    osint_case_results = osint_case.get_results()
+
+    # Print out all the results
+    for result in other_case_results, recon_case_results, osint_case_results:
+        pprint(result)

requirements.txt

@@ -0,0 +1,5 @@
+certifi==2020.6.20
+chardet==3.0.4
+idna==2.10
+requests==2.24.0
+urllib3==1.25.9

src/core/base/base.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+"""
+Defines base runner
+"""
+from logging import getLogger
+from typing import Any
+
+
+class BaseRunner:
+    """
+    Defines base runner class to create childs from
+    """
+
+    def __init__(self, logger: str = __name__):
+        """
+        Initialize the base class
+        :param logger: logger name
+        """
+        self.log = getLogger(name=logger)
+        self.log.info("script started")
+
+    def run(self, *args, **kwargs) -> Any:
+        """
+        Base run method
+        :param args: some positional args
+        :param kwargs: some named args
+        :return: None in this case (base class)
+        """
+        pass

src/core/base/osint.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+"""
+Defines base runner for the OSINT scripts
+"""
+from typing import Any
+
+from src.core.utils.validators import validate_kwargs
+from src.core.base.base import BaseRunner
+
+
+class PossibleKeys:
+    """
+    Defines default values for the function arguments (kwargs, named args)
+    """
+
+    KEYS = ["email", "username", "fullname", "vk_api_key"]
+
+
+class OsintRunner(BaseRunner):
+    def __init__(self, logger: str = __name__):
+        super(OsintRunner, self).__init__(logger)
+
+    @validate_kwargs(PossibleKeys.KEYS)
+    def run(self, *args, **kwargs) -> Any:
+        """
+        OsintRunner 'run' base implementation
+        :param args: some positional args
+        :param kwargs: some named args
+        :return: None in this case(base class)
+        """
+        pass

src/core/base/recon.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+
+"""
+Defines base runner for the RECON scripts
+"""
+from typing import Any
+
+from src.core.utils.validators import validate_kwargs
+from src.core.base.base import BaseRunner
+
+
+class PossibleKeys:
+    """
+    Defines default values for the function arguments (kwargs, named args)
+    """
+
+    KEYS = ["host", "hostname", "ip", "url"]
+
+
+class ReconRunner(BaseRunner):
+    def __init__(self, logger: str = __name__):
+        super(ReconRunner, self).__init__(logger)
+
+    @validate_kwargs(PossibleKeys.KEYS)
+    def run(self, *args, **kwargs) -> Any:
+        """
+        ReconRunner 'run' base implementation
+        :param args: some positional args
+        :param kwargs: some named args
+        :return: None in this case(base class)
+        """
+        pass

src/core/case/base.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+from src.core.runner.runner import ScriptRunner
+
+
+class BaseCase:
+    """
+    Defines base search case
+    """
+
+    def __init__(self, category: str = "other", *args, **kwargs):
+        """
+        Init base
+        :param category: default category to search
+        :param args: some args
+        :param kwargs: some kwargs
+        """
+        self.runner = ScriptRunner()
+        self.args = args
+        self.kwargs = kwargs
+        self.category = category
+
+    def process(self, *args, **kwargs) -> dict:
+        """
+        Process results
+        :param args: some args
+        :param kwargs: some kwargs
+        :return: results
+        """
+        _args = args or self.args
+        _kwargs = kwargs or self.kwargs
+        self.runner.run_category(category=self.category, *_args, **_kwargs)
+        return self.runner.get_results()
+
+    def get_results(self) -> dict:
+        """
+        Return results
+        :return: dict with results
+        """
+        return self.runner.get_results()

src/core/case/osint.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+from src.core.case.base import BaseCase
+
+
+class OsintCase(BaseCase):
+    def __init__(self, *args, **kwargs):
+        super(OsintCase, self).__init__(category="osint", *args, **kwargs)

src/core/case/recon.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+from src.core.case.base import BaseCase
+
+
+class ReconCase(BaseCase):
+    def __init__(self, *args, **kwargs):
+        super(ReconCase, self).__init__(category="recon", *args, **kwargs)

src/core/exceptions/.gitkeep

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+"""
+Defines basic scripts runner
+"""
+
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from importlib.machinery import SourceFileLoader
+from logging import basicConfig, INFO
+from pathlib import Path
+from types import ModuleType
+
+from src.core.utils.response import ScriptResponse
+
+basicConfig(level=INFO)
+
+
+class Defaults:
+    SCRIPTS_BASE = Path("src/scripts/")
+
+
+class ScriptRunnerPaths:
+    OSINT = Defaults.SCRIPTS_BASE.joinpath("osint")
+    RECON = Defaults.SCRIPTS_BASE.joinpath("recon")
+    CONVERT = Defaults.SCRIPTS_BASE.joinpath("convert")
+    OTHER = Defaults.SCRIPTS_BASE.joinpath("other")
+
+
+class ScriptRunner:
+    def __init__(self):
+        self.scripts = {}
+        self.results = {}
+
+    def exec_script(
+        self,
+        path: str or Path,
+        script_class: str = "Runner",
+        function: str = "run",
+        args: list or None = None,
+        kwargs: dict or None = None,
+    ) -> ScriptResponse:
+        """
+        Load and exec python script
+        :param path: name of the script to load
+        :param script_class: class to initialize when script is started
+        :param function: name of the function to run from script
+        :param args: args to pass into the module
+        :param kwargs: kwargs to pass into the module
+        :return: result of the function
+        """
+        if args is None:
+            args = []
+        if kwargs is None:
+            kwargs = {}
+        loader = SourceFileLoader(fullname=script_class, path=str(path))
+        module = ModuleType(name=loader.name)
+        loader.exec_module(module)
+        class_instance = getattr(module, script_class)(logger=path.parent.stem)
+        try:
+            result = getattr(class_instance, function)(*args, **kwargs)
+        except Exception as unexp_err:
+            result = ScriptResponse.error(message=str(unexp_err))
+        result.update({"script": Path(path).parent.stem})
+        return result
+
+    def get_scripts(self) -> dict:
+        """
+        Load the dictionary with all of the scripts
+        :return: dict {'dir':['script1', 'script2', ...]}
+        """
+        for directory in [
+            ScriptRunnerPaths.OSINT,
+            ScriptRunnerPaths.RECON,
+            ScriptRunnerPaths.OTHER,
+            ScriptRunnerPaths.CONVERT,
+        ]:
+            self.scripts.update({directory.stem: list()})
+            for file in directory.glob("*/__main__.py"):
+                self.scripts[directory.stem].append(file)
+        return self.scripts
+
+    def run_category(self, category: str, *args, **kwargs) -> None:
+        """
+        Run a category with scripts
+        :return: nothing
+        """
+        if not self.scripts:
+            self.get_scripts()
+        futures = []
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            for script in self.scripts.get(category, []):
+                futures.append(
+                    executor.submit(
+                        fn=partial(
+                            self.exec_script, path=script, args=args, kwargs=kwargs
+                        )
+                    )
+                )
+        for future in futures:
+            result = future.result()
+            self.results.update({result.get("script"): result})
+
+    def get_results(self) -> dict:
+        """
+        Return retrieved results
+        :return:
+        """
+        return self.results