Merge pull request #16 from echang1802/package-setup

Package setup

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

setup.py

@@ -0,0 +1,28 @@
+import setuptools
+
+with open("README.md", "r", encoding="utf-8") as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name='normandy',
+    version='0.2',
+    author='Eloy Chang',
+    author_email="[email protected]",
+    description='A data pipeline framework.',
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url="https://github.com/echang1802/normandy",
+    package_dir={"": "src"},
+    packages=setuptools.find_packages(where="src"),
+    python_requires=">=3.6",
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    entry_points={
+        'console_scripts': [
+            'normandy = normandy.normandy:run'
+        ]
+    }
+)

src/normandy/__init__.py

@@ -0,0 +1 @@
+from .normandy import run

src/normandy/engine/__init__.py

@@ -0,0 +1,7 @@
+from .errors import test_error
+from .errors import step_error
+from .errors import excecution_error
+from .logger import main_logger
+from .logger import logger
+from .pipeline import pipeline
+from .variables_storage import variables_storage

engine/errors.py → src/normandy/engine/errors.py

@@ -13,3 +13,9 @@ class step_error(Error):
 
     def __init__(self, message):
         self.message = message
+
+class excecution_error(Error):
+    # Step exception without errors tolerance
+
+    def __init__(self, message):
+        self.message = message

engine/pipeline.py → src/normandy/engine/pipeline.py

@@ -3,7 +3,7 @@ class pipeline:
 
     from datetime import datetime
     from multiprocessing import Pool
-    from engine.logger import main_logger, logger
+    from normandy.engine.logger import main_logger, logger
 
     def __init__(self, env, tags):
         from yaml import load
@@ -17,6 +17,12 @@ def __init__(self, env, tags):
             self.__confs__["active_env"] = env
             self.__log_level__ = 0
 
+    def set_path(self):
+        import os
+
+        os.chdir(self.__confs__["path"])
+        return
+
     def get_env_confs(self):
         return self.__confs__["envs"][self.__confs__["active_env"]]
 
@@ -108,14 +114,16 @@ def __setup__(self, data, setup):
 
                 def execute(self, pipe):
                     from datetime import datetime
-                    from importlib import import_module
-                    from engine.errors import step_error
-                    from engine.logger import logger
+                    from importlib.util import spec_from_file_location, module_from_spec
+                    from normandy.engine.errors import step_error
+                    from normandy.engine.logger import logger
 
                     _t = datetime.now()
                     log = logger(self, pipe.__log_level__)
 
-                    module = import_module(f"pipeline.{self.__from_step__}.{self.__name__}")
+                    spec = spec_from_file_location(f"{self.__from_step__}.{self.__name__}", f"{pipe.__confs__['path']}/pipeline/{self.__from_step__}/{self.__name__}.py")
+                    module = module_from_spec(spec)
+                    spec.loader.exec_module(module)
                     try:
                         exit_vars = module.process(pipe, log)
                     except Exception as e:

src/normandy/normandy.py

@@ -0,0 +1,71 @@
+
+import os
+import yaml
+import click
+from normandy.engine.pipeline import pipeline
+from normandy.engine.errors import excecution_error
+
+def rur_pipe(tags, env):
+    pipe = pipeline(env = env, tags = tags)
+    pipe.start_pipeline()
+
+def create_framework(project_path):
+    actual_path = os.getcwd()
+    try:
+        os.chdir(project_path)
+    except:
+        os.mkdir(project_path)
+    finally:
+        os.chdir(project_path)
+
+    # Create pieline folder and basic confs
+    os.mkdir("pipeline")
+    os.mkdir("pipeline/extract")
+    os.mkdir("pipeline/transform")
+    os.mkdir("pipeline/load")
+    basic_confs = {
+        "flows" : {
+            "my-flow" : {
+                "tags" : ["deafult"],
+                "steps" : {
+                    "extract" : ["extract_data"],
+                    "transform" : ["transform_data"],
+                    "load" : ["load_data"]
+                }
+            }
+        },
+        "confs" : {
+            "envs" : {
+                "dev" : "dev confs",
+                "prod" : "prod confs",
+            }
+        }
+    }
+    with open("pipeline/pipeline_conf.yml", "w") as file:
+        yaml.dump(basic_confs, file, default_flow_style = False)
+
+    # Create extra folders
+    os.mkdir("logs")
+    os.mkdir("temp")
+
+    os.chdir(actual_path)
+    print("Normandy folder structure created")
+
+
+@click.command()
+@click.option("--create-project", "create_project", is_flag = True, help = "Used to create a Normandy project structure in the specify directory.")
+@click.option("-project-path", "project_path", default = None, help = "Path where to start Normandy project.")
+@click.option("--run-pipeline", "run_pipeline", is_flag = True, help = "Execute the pipeline.")
+@click.option("-tags", default = ["default"], help = "Flows with this tag will run.", show_default = True, multiple=True)
+@click.option("-env", default = "dev", help = "Enviroment to run.", show_default=True)
+def run(create_project, project_path, run_pipeline, tags, env):
+    if create_project and run_pipeline:
+        raise excecution_error("Cannot use create-project and run-pipeline at the same time")
+    if run_pipeline:
+        rur_pipe(tags, env)
+    elif create_project:
+        if project_path is None:
+            raise excecution_error("Project path must be specify")
+        create_framework(project_path)
+    else:
+        raise excecution_error("No option selected")