Merge pull request #103 from dlt-hub/rfix/fixes-module-import

fixes module import

Author: rudolfix

dlt/cli/_dlt.py

@@ -55,7 +55,7 @@ def deploy_command_wrapper(pipeline_script_path: str, deployment_method: str, sc
         click.echo("Currently you must run the pipeline at least once")
     except InvalidGitRepositoryError:
         click.secho(
-            "The path %s is invalid, outside of any git repository or git repository is not initialized. Initialize the repository and add it to Github by following the guide %s" % (pipeline_script_path, fmt.bold("https://docs.github.com/en/get-started/quickstart/create-a-repo")),
+            "No git repository found for pipeline script %s. Add your local code to Github as described here: %s" % (fmt.bold(pipeline_script_path), fmt.bold("https://docs.github.com/en/get-started/importing-your-projects-to-github/importing-source-code-to-github/adding-locally-hosted-code-to-github")),
             err=True,
             fg="red"
         )

dlt/cli/deploy_command.py

@@ -45,7 +45,7 @@ def deploy_command(pipeline_script_path: str, deployment_method: str, schedule:
         if "github.com" not in origin:
             raise CliCommandException("deploy", f"Your current repository origin is not set to github but to {origin}.\nYou must change it to be able to run the pipelines with github actions: https://docs.github.com/en/get-started/getting-started-with-git/managing-remote-repositories")
     except ValueError:
-        raise CliCommandException("deploy", "Your current repository has no origin set. Please set it up to be able to run the pipelines with github actions: https://docs.github.com/en/get-started/getting-started-with-git/managing-remote-repositories")
+        raise CliCommandException("deploy", "Your current repository has no origin set. Please set it up to be able to run the pipelines with github actions: https://docs.github.com/en/get-started/importing-your-projects-to-github/importing-source-code-to-github/adding-locally-hosted-code-to-github")
 
     # convert to path relative to repo
     repo_pipeline_script_path = repo_storage.from_wd_to_relative_path(pipeline_script_path)

dlt/cli/init_command.py

@@ -183,7 +183,7 @@ def init_command(pipeline_name: str, destination_name: str, use_generic_template
     )
 
     # inspect the script
-    script_module = inspect_pipeline_script(clone_storage.make_full_path(init_script_name))
+    script_module = inspect_pipeline_script(clone_storage.storage_path, clone_storage.to_relative_path(init_script_name))
 
     if len(_SOURCES) == 0:
         raise CliCommandException("init", f"The pipeline script {init_script_name} is not creating or importing any sources or resources")

dlt/cli/utils.py

@@ -1,5 +1,6 @@
 import ast
 import os
+from pathlib import Path
 import sys
 import tempfile
 from importlib import import_module
@@ -32,11 +33,19 @@ def clone_command_repo(command: str, branch: str) -> FileStorage:
 def load_command_module(template_dir: str) -> ModuleType:
     # import the settings from the clone
     template_dir, template_module_name = os.path.split(template_dir.rstrip("/"))
-    sys.path.append(template_dir)
+    module_path, package = os.path.split(template_dir)
+    module, _ = os.path.splitext(template_module_name)
+    module = ".".join(Path(module).parts)
+
+    sys_path: str = None
+    if module_path not in sys.path:
+        sys_path = module_path
+        # path must be first so we always load our module of
+        sys.path.insert(0, sys_path)
     try:
-        return import_module(template_module_name)
+        return import_module(f"{package}.{module}")
     finally:
-        sys.path.remove(template_dir)
+        sys.path.remove(sys_path)
 
 
 def parse_init_script(command: str, script_source: str, init_script_name: str) -> PipelineScriptVisitor:

dlt/reflection/script_inspector.py

@@ -1,10 +1,12 @@
 import os
 import sys
+from pathlib import Path
 from types import ModuleType
 from typing import Any, Tuple
 from unittest.mock import patch
 from importlib import import_module
 
+from dlt.common import logger
 from dlt.common.exceptions import DltException
 from dlt.common.typing import DictStrAny
 
@@ -16,14 +18,18 @@ def patch__init__(self: Any, *args: Any, **kwargs: Any) -> None:
     raise PipelineIsRunning(self, args, kwargs)
 
 
-def inspect_pipeline_script(script_path: str) -> ModuleType:
+def inspect_pipeline_script(module_path:str, script_relative_path: str) -> ModuleType:
+    if os.path.isabs(script_relative_path):
+        raise ValueError(script_relative_path, f"Not relative path to {module_path}")
+    script_path = os.path.join(module_path, script_relative_path)
     if not os.path.isfile(script_path):
         raise FileNotFoundError(script_path)
 
     # get module import data
-    path, module = os.path.split(script_path)
-    _, package = os.path.split(path)
-    module, _ = os.path.splitext(module)
+    path, package = os.path.split(module_path)
+    # _, package = os.path.split(path)
+    module, _ = os.path.splitext(script_relative_path)
+    module = ".".join(Path(module).parts)
 
     # add path to module search
     sys_path: str = None
@@ -35,6 +41,7 @@ def inspect_pipeline_script(script_path: str) -> ModuleType:
 
         # patch entry points to pipeline, sources and resources to prevent pipeline from running
         with patch.object(Pipeline, '__init__', patch__init__), patch.object(DltSource, '__init__', patch__init__), patch.object(PipeIterator, '__init__', patch__init__):
+            print(f"Importing pipeline script from path {path} and module: {package}.{module}")
             return import_module(f"{package}.{module}")
     finally:
         # remove script module path

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-dlt"
-version = "0.2.0a5"
+version = "0.2.0a6"
 description = "DLT is an open-source python-native scalable data loading framework that does not require any devops efforts to run."
 authors = ["dltHub Inc. <[email protected]>"]
 maintainers = [ "Marcin Rudolf <[email protected]>", "Adrian Brudaru <[email protected]>",]

tests/cli/test_cli.py

@@ -35,11 +35,11 @@ def test_init_command_template() -> None:
         assert_init_files("debug_pipeline", "bigquery")
 
 
-def test_init_command_chess() -> None:
+def test_init_command_chess_verified_pipeline() -> None:
     _SOURCES.clear()
 
     with set_working_dir(TEST_STORAGE_ROOT):
-        init_command.init_command("chess_pipeline", "postgres", False)
+        init_command.init_command("chess", "postgres", False)
         assert_init_files("chess_pipeline", "postgres")
 
 

tests/common/test_git.py

@@ -5,7 +5,7 @@
 from dlt.common.storages import FileStorage
 from dlt.common.git import clone_repo, ensure_remote_head, git_custom_key_command
 
-from tests.utils import test_storage
+from tests.utils import test_storage, skipifwindows
 from tests.common.utils import load_secret, modify_and_commit_file, restore_secret_storage_path
 
 
@@ -62,6 +62,7 @@ def test_clone_with_deploy_key_access_denied(test_storage: FileStorage) -> None:
             clone_repo(PRIVATE_REPO, repo_path, with_git_command=git_command)
 
 
+@skipifwindows
 def test_clone_with_deploy_key(test_storage: FileStorage) -> None:
     secret = load_secret("deploy_key")
     repo_path = test_storage.make_full_path("private_repo_access")
@@ -70,6 +71,7 @@ def test_clone_with_deploy_key(test_storage: FileStorage) -> None:
         ensure_remote_head(repo_path, with_git_command=git_command)
 
 
+@skipifwindows
 def test_repo_status_update(test_storage: FileStorage) -> None:
     secret = load_secret("deploy_key")
     repo_path = test_storage.make_full_path("private_repo_access")

tests/utils.py

@@ -124,3 +124,7 @@ def assert_no_dict_key_starts_with(d: StrAny, key_prefix: str) -> None:
 skipifnotwindows = pytest.mark.skipif(
     platform.system() != "Windows", reason="runs only on windows"
 )
+
+skipifwindows = pytest.mark.skipif(
+    platform.system() == "Windows", reason="does not runs on windows"
+)