[IMP] Relocating plugin files to the configuration directory (#120)

* [IMP] Relocating plugin files to the configuration directory

* [REF] Switch folder to python package (move out plugins of odev path)

* [FIX] Attempt to fix "module not found error"

* [REF] Add importlib and attempt to import plugin modules during loading.

* [REM] Clean useless import mechanism

* [REF] Replace dynamic plugin loading with explicit `__all__` exports in common modules.

* [FIX] Clean dynamic import code

* [FIX] Import lib is still needed

* [FIX] Check ipdb in the new plugin folder

* [IMP] Bump version / Move upgrade script

* [IMP] Add back contextlib

Author: sea-odoo

.gitignore

@@ -18,7 +18,6 @@ htmlcov
 coverage.xml
 
 # --- Odev plugins, loaded as submodules
-odev/plugins/*
 tests/plugins/*
 
 # --- Temporary and testing files

install.sh

@@ -34,11 +34,14 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
+echo "Creating plugins directory"
+mkdir -p ~/.config/odev/plugins
+
 echo "Installing dependencies"
 ~/.config/odev/venv/bin/pip install -r requirements.txt > /dev/null 2>&1
 ~/.config/odev/venv/bin/pip install -r requirements-dev.txt > /dev/null 2>&1
 
-find odev/plugins/*/ -type f -name 'requirements.txt' | while read reqfile; do
+find ~/.config/odev/plugins/*/ -type f -name 'requirements.txt' | while read reqfile; do
     ~/.config/odev/venv/bin/pip install -r "$reqfile" > /dev/null 2>&1
     if [ $? -ne 0 ]; then
         echo "Failed to install dependencies from $reqfile"

odev/_version.py

@@ -22,4 +22,4 @@
 # or merged change.
 # ------------------------------------------------------------------------------
 
-__version__ = "4.23.0"
+__version__ = "4.24.0"

odev/common/__init__.py

@@ -1,42 +1,7 @@
 """Common utilities for odev."""
 
-import pkgutil
-from importlib import import_module
-from pathlib import Path
-from typing import cast
-
-
-# --- Import helper submodules -------------------------------------------------
-
-from . import actions
 from . import arguments as args
-from . import bash
-from . import config
-from . import console
-from . import debug
-from . import logging
 from . import odev
-from . import odoobin
-from . import postgres
-from . import progress
-from . import python
-from . import signal_handling
-from . import string
-from . import thread
-from . import version
-
-
-# --- Load plugins' helpers ----------------------------------------------------
-
-odev_path = Path(__file__).parents[1]
-
-plugins = [path for path in (odev_path / "plugins").glob("*/common") if path.is_dir()]
-modules = pkgutil.iter_modules([directory.as_posix() for directory in plugins])
-
-for module_info in modules:
-    module_path = cast(str, module_info.module_finder.path).replace(str(odev_path.parent) + "/", "").replace("/", ".")  # type: ignore [union-attr]
-    module = import_module(f"{module_path}.{module_info.name}")
-
 
 # --- Setup the framework and make it globally available -----------------------
 

odev/common/commands/__init__.py

@@ -1,12 +1,6 @@
 """Odev base command classes."""
 
-import pkgutil
-from importlib import import_module
-from pathlib import Path
-from inspect import isclass
-
-# --- Common modules -----------------------------------------------------------
-from typing import TypeVar, cast
+from typing import TypeVar
 
 from .base import Command, CommandError
 from .database import DatabaseCommand, DatabaseOrRepositoryCommand, LocalDatabaseCommand, RemoteDatabaseCommand
@@ -19,23 +13,20 @@
     OdoobinTemplateCommand,
 )
 
-
-# --- Plugins ------------------------------------------------------------------
-
-odev_path = Path(__file__).parent.parent.parent
-
-plugins = [path for path in (odev_path / "plugins").glob("*/common/commands") if path.is_dir()]
-modules = pkgutil.iter_modules([directory.as_posix() for directory in plugins])
-
-for module_info in modules:
-    module_path = cast(str, module_info.module_finder.path).replace(str(odev_path.parent) + "/", "").replace("/", ".")  # type: ignore [union-attr]
-    module = import_module(f"{module_path}.{module_info.name}")
-
-    for attribute in dir(module):
-        obj = getattr(module, attribute)
-
-        if isclass(obj) and issubclass(obj, Command) and obj is not Command:
-            globals()[attribute] = obj
-
-
 CommandType = TypeVar("CommandType", Command, DatabaseCommand)
+
+__all__ = [
+    "TEMPLATE_SUFFIX",
+    "Command",
+    "CommandError",
+    "CommandType",
+    "DatabaseCommand",
+    "DatabaseOrRepositoryCommand",
+    "GitCommand",
+    "LocalDatabaseCommand",
+    "OdoobinCommand",
+    "OdoobinShellCommand",
+    "OdoobinShellScriptCommand",
+    "OdoobinTemplateCommand",
+    "RemoteDatabaseCommand",
+]

odev/common/config.py

@@ -293,8 +293,8 @@ def __init_sections(self):
 
         modules = [inspect.getmodule(self)]
 
-        # Find all odev/plugins/*/config.py files and import their Section subclasses
-        plugins_config_paths = glob.glob(str(Path(__file__).parent.parent / "plugins" / "*" / "config.py"))
+        # Find all ~/.config/odev/plugins/*/config.py files and import their Section subclasses
+        plugins_config_paths = glob.glob(str(CONFIG_DIR / "plugins" / "*" / "config.py"))
 
         for config_path in plugins_config_paths:
             module_name = f"odev.plugins.{Path(config_path).parent.name}.config"

odev/common/connectors/__init__.py

@@ -1,32 +1,17 @@
 """Connectors to external services."""
 
-import pkgutil
-from importlib import import_module
-from pathlib import Path
-from typing import cast
-from inspect import isclass
-
-# --- Common modules -----------------------------------------------------------
 from .base import Connector
 from .git import GitConnector, GitWorktree, Stash
 from .postgres import PostgresConnector
 from .rest import RestConnector
 from .rpc import RpcConnector
 
-
-# --- Plugins ------------------------------------------------------------------
-
-odev_path = Path(__file__).parent.parent.parent
-
-plugins = [path for path in (odev_path / "plugins").glob("*/common/connectors") if path.is_dir()]
-modules = pkgutil.iter_modules([directory.as_posix() for directory in plugins])
-
-for module_info in modules:
-    module_path = cast(str, module_info.module_finder.path).replace(str(odev_path.parent) + "/", "").replace("/", ".")  # type: ignore [union-attr]
-    module = import_module(f"{module_path}.{module_info.name}")
-
-    for attribute in dir(module):
-        obj = getattr(module, attribute)
-
-        if isclass(obj) and issubclass(obj, Connector) and obj is not Connector:
-            globals()[attribute] = obj
+__all__ = [
+    "Connector",
+    "GitConnector",
+    "GitWorktree",
+    "PostgresConnector",
+    "RestConnector",
+    "RpcConnector",
+    "Stash",
+]

odev/common/databases/__init__.py

@@ -1,30 +1,15 @@
 """Database handling."""
 
-import pkgutil
-from importlib import import_module
-from pathlib import Path
-from typing import cast
-from inspect import isclass
-
-# --- Common modules -----------------------------------------------------------
 from .base import Branch, Database, DummyDatabase, Filestore, Repository
 from .local import LocalDatabase
 from .remote import RemoteDatabase
 
-
-# --- Plugins ------------------------------------------------------------------
-
-odev_path = Path(__file__).parent.parent.parent
-
-plugins = [path for path in (odev_path / "plugins").glob("*/common/databases") if path.is_dir()]
-modules = pkgutil.iter_modules([directory.as_posix() for directory in plugins])
-
-for module_info in modules:
-    module_path = cast(str, module_info.module_finder.path).replace(str(odev_path.parent) + "/", "").replace("/", ".")  # type: ignore [union-attr]
-    module = import_module(f"{module_path}.{module_info.name}")
-
-    for attribute in dir(module):
-        obj = getattr(module, attribute)
-
-        if isclass(obj) and issubclass(obj, Database) and obj is not Database:
-            globals()[attribute] = obj
+__all__ = [
+    "Branch",
+    "Database",
+    "DummyDatabase",
+    "Filestore",
+    "LocalDatabase",
+    "RemoteDatabase",
+    "Repository",
+]

odev/common/debug.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 
 from odev.common import bash, string
+from odev.common.config import CONFIG_DIR
 from odev.common.logging import logging
 
 
@@ -41,7 +42,13 @@ def find_debuggers(root: str | Path) -> Generator[tuple[Path, int], None, None]:
 
 # ------------------------------------------------------------------------------
 # Find calls to interactive debuggers within odev's source code
-debuggers = [f"{file.as_posix()}:{line}" for file, line in find_debuggers(Path(__file__).parents[1])]
+sources = [Path(__file__).parents[1]]
+plugins_path = CONFIG_DIR / "plugins"
+
+if plugins_path.is_dir():
+    sources.append(plugins_path)
+
+debuggers = [f"{file.as_posix()}:{line}" for source in sources for file, line in find_debuggers(source)]
 
 if debuggers:
     logger.warning(f"Interactive debuggers detected:\n{string.join_bullet(debuggers)}")

odev/common/odev.py

@@ -1,6 +1,7 @@
 """Self update Odev by pulling latest changes from the git repository."""
 
 import contextlib
+import importlib
 import inspect
 import os
 import pkgutil
@@ -36,7 +37,7 @@
 from odev.common import progress, string
 from odev.common.commands import CommandType
 from odev.common.commands.database import DatabaseType
-from odev.common.config import Config
+from odev.common.config import CONFIG_DIR, Config
 from odev.common.connectors.git import GitConnector, Stash
 from odev.common.console import Console, console
 from odev.common.errors import OdevError
@@ -184,7 +185,7 @@ def tests_path(self) -> Path:
     @property
     def plugins_path(self) -> Path:
         """Local path to the plugins directory."""
-        return self.base_path / "plugins"
+        return CONFIG_DIR / "plugins"
 
     @property
     def commands_path(self) -> Path:
@@ -516,8 +517,12 @@ def import_commands(self, sources: Iterable[Path]) -> list[type[CommandType]]:
             if not isinstance(module_info.module_finder, FileFinder):
                 raise TypeError("Module finder is not a FileFinder instance")
 
-            module_path = Path(module_info.module_finder.path) / f"{module_info.name}.py"
-            spec = spec_from_file_location(module_path.stem, module_path.as_posix())
+            if module_info.ispkg:
+                module_path = Path(module_info.module_finder.path) / module_info.name / "__init__.py"
+            else:
+                module_path = Path(module_info.module_finder.path) / f"{module_info.name}.py"
+
+            spec = spec_from_file_location(module_info.name, module_path.as_posix())
 
             if spec is None or spec.loader is None:
                 raise ImportError(f"Cannot load module {module_info.name} from {module_path.as_posix()}")
@@ -566,11 +571,29 @@ def register_plugin_commands(self) -> None:
 
     def _register_plugin_commands(self) -> None:
         """Register all commands from the plugins directories."""
+        # Ensure odev.plugins exists as a module so legacy imports work
+        odev_module = sys.modules.get("odev")
+        if odev_module:
+            if not hasattr(odev_module, "plugins"):
+                odev_module.plugins = ModuleType("odev.plugins")
+                odev_module.plugins.__path__ = []
+                sys.modules["odev.plugins"] = odev_module.plugins
+
+            if str(self.plugins_path) not in odev_module.plugins.__path__:
+                odev_module.plugins.__path__.append(str(self.plugins_path))
+
         for plugin in self.plugins:
             logger.debug(
                 f"Loading plugin {plugin.name!r} version {string.stylize(plugin.manifest['version'], 'repr.version')}"
             )
 
+            self._install_plugin_requirements(plugin.path)
+
+            try:
+                importlib.import_module(f"odev.plugins.{plugin.path.name}")
+            except ImportError as error:
+                logger.debug(f"Could not import plugin module {plugin.path.name}: {error}")
+
             for command_class in self.import_commands(plugin.path.glob("commands/**")):
                 command_names = [command_class._name] + (list(command_class._aliases) or [])
                 base_command_class = self.commands.get(command_class._name)