allow to override commands once again

fixes #3705
fixes #3697

Author: Vladimir Kotal

tools/src/main/python/opengrok_tools/scm/cvs.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
@@ -29,8 +29,8 @@
 
 
 class CVSRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(name, logger, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('cvs'))
 
@@ -39,8 +39,8 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
 
     def reposync(self):
         hg_command = [self.command, "update", "-dP"]
-        cmd = self.getCommand(hg_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(hg_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         self.logger.info("output of {}:".format(cmd))
         self.logger.info(cmd.getoutputstr())

tools/src/main/python/opengrok_tools/scm/git.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
@@ -29,25 +29,28 @@
 
 
 class GitRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(name, logger, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('git'))
 
         if not self.command:
             raise RepositoryException("Cannot get git command")
 
+    def _configure_git_pull(self):
         # The incoming() check relies on empty output so configure
         # the repository first to avoid getting extra output.
         git_command = [self.command, "config", "--local", "pull.ff", "only"]
-        cmd = self.getCommand(git_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(git_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         if cmd.getretcode() != 0 or cmd.getstate() != Command.FINISHED:
             cmd.log_error("failed to configure git pull.ff")
 
     def reposync(self):
+        self._configure_git_pull()
         return self._run_custom_sync_command([self.command, 'pull', '--ff-only'])
 
     def incoming_check(self):
+        self._configure_git_pull()
         return self._run_custom_incoming_command([self.command, 'pull', '--dry-run'])

tools/src/main/python/opengrok_tools/scm/mercurial.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
@@ -29,8 +29,8 @@
 
 
 class MercurialRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(name, logger, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('hg'))
 
@@ -39,8 +39,8 @@ def __init__(self, logger, path, project, command, env, hooks, timeout):
 
     def get_branch(self):
         hg_command = [self.command, "branch"]
-        cmd = self.getCommand(hg_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(hg_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         self.logger.info("output of {}:".format(cmd))
         self.logger.info(cmd.getoutputstr())
@@ -68,8 +68,8 @@ def reposync(self):
         if branch != "default":
             hg_command.append("-b")
             hg_command.append(branch)
-        cmd = self.getCommand(hg_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(hg_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         self.logger.info("output of {}:".format(cmd))
         self.logger.info(cmd.getoutputstr())
@@ -82,8 +82,8 @@ def reposync(self):
         # some servers do not support it.
         if branch == "default":
             hg_command.append("--check")
-        cmd = self.getCommand(hg_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(hg_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         self.logger.info("output of {}:".format(cmd))
         self.logger.info(cmd.getoutputstr())
@@ -104,8 +104,8 @@ def incoming_check(self):
         if branch != "default":
             hg_command.append("-b")
             hg_command.append(branch)
-        cmd = self.getCommand(hg_command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(hg_command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         self.logger.info("output of {}:".format(cmd))
         self.logger.info(cmd.getoutputstr())

tools/src/main/python/opengrok_tools/scm/perforce.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright 2020 Robert Williams
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
@@ -29,8 +29,8 @@
 
 
 class PerforceRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(name, logger, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('p4'))
 

tools/src/main/python/opengrok_tools/scm/repo.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
@@ -28,8 +28,8 @@
 
 
 class RepoRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(logger, name, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('repo'))
 

tools/src/main/python/opengrok_tools/scm/repofactory.py

@@ -33,16 +33,11 @@
 from .teamware import TeamwareRepository
 
 
-# Note: these have to correspond with get_repository().
-REPO_TYPES = ["mercurial", "hg", "teamware", "sccs", "cvs",
-              "svn", "subversion", "git", "perforce", "repo"]
-
-
 def get_repository(path, repo_type, project,
                    commands=None, env=None, hooks=None, timeout=None):
     """
-    :param path: full path
-    :param repo_type: repository type
+    :param path: full path for the working directory
+    :param repo_type: repository type name
     :param project: project name
     :param commands: commands dictionary with paths to SCM utilities
     :param env: environment variables dictionary
@@ -56,37 +51,38 @@ def get_repository(path, repo_type, project,
 
     repo_lower = repo_type.lower()
 
-    logger.debug("Constructing repo object for path {}".format(path))
+    logger.debug("Constructing repository object of type '{}' for path '{}'".
+                 format(repo_type, path))
 
     if not commands:
         commands = {}
 
     if repo_lower in ["mercurial", "hg"]:
-        return MercurialRepository(logger, path, project,
+        return MercurialRepository(repo_type, logger, path, project,
                                    commands.get("hg"),
                                    env, hooks, timeout)
     elif repo_lower in ["teamware", "sccs"]:
-        return TeamwareRepository(logger, path, project,
+        return TeamwareRepository(repo_type, logger, path, project,
                                   commands.get("teamware"),
                                   env, hooks, timeout)
     elif repo_lower == "cvs":
-        return CVSRepository(logger, path, project,
+        return CVSRepository(repo_type, logger, path, project,
                              commands.get("cvs"),
                              env, hooks, timeout)
     elif repo_lower in ["svn", "subversion"]:
-        return SubversionRepository(logger, path, project,
+        return SubversionRepository(repo_type, logger, path, project,
                                     commands.get("svn"),
                                     env, hooks, timeout)
     elif repo_lower == "git":
-        return GitRepository(logger, path, project,
+        return GitRepository(repo_type, logger, path, project,
                              commands.get("git"),
                              env, hooks, timeout)
     elif repo_lower == "perforce":
-        return PerforceRepository(logger, path, project,
+        return PerforceRepository(repo_type, logger, path, project,
                                   commands.get("perforce"),
                                   env, hooks, timeout)
     elif repo_lower == "repo":
-        return RepoRepository(logger, path, project,
+        return RepoRepository(repo_type, logger, path, project,
                               commands.get("repo"),
                               env, hooks, timeout)
     else:

tools/src/main/python/opengrok_tools/scm/repository.py

@@ -18,11 +18,12 @@
 #
 
 #
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
 import abc
+import os
 
 from ..utils.command import Command
 
@@ -43,9 +44,11 @@ class Repository:
 
     SYNC_COMMAND_SECTION = 'sync'
     INCOMING_COMMAND_SECTION = 'incoming'
+    COMMAND_PROPERTY = 'command'
 
-    def __init__(self, logger, path, project, configured_commands, env, hooks,
-                 timeout):
+    def __init__(self, name, logger, path, project, configured_commands, env, hooks, timeout):
+        self.name = name
+        self.command = None
         self.logger = logger
         self.path = path
         self.project = project
@@ -59,12 +62,16 @@ def __init__(self, logger, path, project, configured_commands, env, hooks,
     def __str__(self):
         return self.path
 
-    def getCommand(self, cmd, **kwargs):
+    def get_command(self, cmd, **kwargs):
+        """
+        :param cmd: command
+        :param kwargs: dictionary of command attributes
+        :return: Command object ready for execution.
+        """
         kwargs['timeout'] = self.timeout
         return Command(cmd, **kwargs)
 
     def sync(self):
-        # Eventually, there might be per-repository hooks added here.
         if self.is_command_overridden(self.configured_commands, self.SYNC_COMMAND_SECTION):
             return self._run_custom_sync_command(
                 self.listify(self.configured_commands[self.SYNC_COMMAND_SECTION])
@@ -77,6 +84,8 @@ def reposync(self):
         Synchronize the repository by running sync command specific for
         given repository type.
 
+        This method definition has to be overriden by given repository class.
+
         Return 1 on failure, 0 on success.
         """
         raise NotImplementedError()
@@ -96,6 +105,8 @@ def incoming(self):
     def incoming_check(self):
         """
         Check if there are any incoming changes.
+        Normally this method definition is overriden, unless the repository
+        type has no way how to check for incoming changes.
 
         Return True if so, False otherwise.
         """
@@ -137,8 +148,8 @@ def _run_command(self, command):
                     - status: 0 on success execution, non-zero otherwise
                     - output: command output as string
         """
-        cmd = self.getCommand(command, work_dir=self.path,
-                              env_vars=self.env, logger=self.logger)
+        cmd = self.get_command(command, work_dir=self.path,
+                               env_vars=self.env, logger=self.logger)
         cmd.execute()
         if cmd.getretcode() != 0 or cmd.getstate() != Command.FINISHED:
             cmd.log_error("failed to perform command")
@@ -164,7 +175,7 @@ def _repository_command(configured_commands, default=lambda: None):
         if isinstance(configured_commands, str):
             return configured_commands
         elif isinstance(configured_commands, dict) and \
-                configured_commands.get('command'):
+                configured_commands.get('command'):  # COMMAND_PROPERTY
             return configured_commands['command']
 
         return default()
@@ -185,3 +196,38 @@ def is_command_overridden(config, command):
         :return: true if overridden, false otherwise
         """
         return isinstance(config, dict) and config.get(command) is not None
+
+    def _check_command(self):
+        """
+        Could be overriden in given repository class to provide different check.
+        :return: True if self.command is a file, False otherwise.
+        """
+        if self.command and not os.path.isfile(self.command):
+            self.logger.error("path for '{}' is not a file: {}".
+                              format(self.name, self.command))
+            return False
+
+        return True
+
+    def check_command(self):
+        """
+        Check the validity of the command. Does not check the command if
+        the sync/incoming is overriden.
+        :return: True if self.command is valid, False otherwise.
+        """
+
+        if isinstance(self.configured_commands, dict):
+            for key in self.configured_commands.keys():
+                if key not in [self.SYNC_COMMAND_SECTION,
+                               self.INCOMING_COMMAND_SECTION,
+                               self.COMMAND_PROPERTY]:
+                    self.logger.error("Unknown property '{}' for '{}'".
+                                      format(key, self.name))
+                    return False
+
+        if self.command and not os.path.exists(self.command):
+            self.logger.error("path for '{}' does not exist: {}".
+                              format(self.name, self.command))
+            return False
+
+        return self._check_command()

tools/src/main/python/opengrok_tools/scm/svn.py

@@ -18,7 +18,7 @@
 #
 
 #
-# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
 # Portions Copyright (c) 2020, Krystof Tulinger <[email protected]>
 #
 
@@ -28,8 +28,8 @@
 
 
 class SubversionRepository(Repository):
-    def __init__(self, logger, path, project, command, env, hooks, timeout):
-        super().__init__(logger, path, project, command, env, hooks, timeout)
+    def __init__(self, name, logger, path, project, command, env, hooks, timeout):
+        super().__init__(name, logger, path, project, command, env, hooks, timeout)
 
         self.command = self._repository_command(command, default=lambda: which('svn'))