[WIP] Autogenerate repos.yaml for missing addons

11.0.Dockerfile

@@ -135,6 +135,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"]
 ONBUILD CMD ["/usr/local/bin/odoo"]
 ONBUILD ARG AGGREGATE=true
 ONBUILD ARG AUTO_REQUIREMENTS=false
+ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git"
+ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git"
 ONBUILD ARG DEPTH_DEFAULT=1
 ONBUILD ARG DEPTH_MERGE=100
 ONBUILD ARG CLEAN=true
@@ -156,6 +158,8 @@ ONBUILD ARG PGPORT=5432
 ONBUILD ARG PGDATABASE=prod
 # Config variables
 ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \
+            DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \
+            DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \
             UNACCENT="$UNACCENT" \
             PGUSER="$PGUSER" \
             PGPASSWORD="$PGPASSWORD" \

8.0.Dockerfile

@@ -121,6 +121,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"]
 ONBUILD CMD ["/usr/local/bin/odoo"]
 ONBUILD ARG AGGREGATE=true
 ONBUILD ARG AUTO_REQUIREMENTS=false
+ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git"
+ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git"
 ONBUILD ARG DEPTH_DEFAULT=1
 ONBUILD ARG DEPTH_MERGE=100
 ONBUILD ARG CLEAN=true
@@ -142,6 +144,8 @@ ONBUILD ARG PGPORT=5432
 ONBUILD ARG PGDATABASE=prod
 # Config variables
 ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \
+            DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \
+            DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \
             UNACCENT="$UNACCENT" \
             PGUSER="$PGUSER" \
             PGPASSWORD="$PGPASSWORD" \

bin/autoaggregate

@@ -1,37 +1,130 @@
-#!/bin/bash
-set -e
-
-conf=/opt/odoo/custom/src/repos
-
-if [ -f "${conf}.yaml" ]; then
-    conf="${conf}.yaml"
-elif [ -f "${conf}.yml" ]; then
-    conf="${conf}.yml"
-fi
-
-# Update linked repositories, if the `repos.yaml` file is found
-if [ -f $conf ]; then
-    log INFO Aggregating repositories from $conf
-    cd $(dirname $conf)
-
-    # Avoid wrong umask in aggregated files
-    if [ -n "$UMASK" ]; then
-        umask "$UMASK"
-    fi
-
-    # Perform aggregation with environment variables expansion
-    set +e
-    gitaggregate --expand-env -c $conf
-    code=$?
-    set -e
-
-    # Avoid wrong user/group in aggregated files
-    if [ -n "$GID" -a -n "$UID" ]; then
-        chown -R "$UID:$GID" .
-    fi
-
-    [ $code -eq 0 ] || exit $code
-else
-    log ERROR Cannot aggregate repositories: $conf not found
-    exit 1
-fi
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import sys
+import yaml
+from multiprocessing import cpu_count
+from subprocess import check_call
+
+from odoobaselib import (
+    ADDONS_YAML,
+    AUTO_REPOS_YAML,
+    CORE,
+    logger,
+    PRIVATE,
+    REPOS_YAML,
+    SRC_DIR,
+)
+
+UMASK = os.environ.get("UMASK")
+UID = int(os.environ.get("UID") or -1)
+GID = int(os.environ.get("GID") or -1)
+DEFAULT_REPO_PATTERN = os.environ.get("DEFAULT_REPO_PATTERN")
+DEFAULT_REPO_PATTERN_ODOO = os.environ.get("DEFAULT_REPO_PATTERN_ODOO")
+
+
+def aggregate(config):
+    """Execute git aggregator to pull git code.
+
+    :param str config:
+        Path where to find the ``repos.yaml`` file.
+    """
+    logger.info("Running gitaggregate with %s", config)
+    old_umask = None
+    try:
+        # Download git code with the specified umask, if any
+        if UMASK:
+            old_umask = os.umask(int(UMASK))
+        check_call(
+            ["gitaggregate", "--expand-env", "--config", config,
+             "--jobs", str(cpu_count() or 1)],
+            cwd=SRC_DIR,
+            stderr=sys.stderr,
+            stdout=sys.stdout,
+        )
+    finally:
+        # Restore umask, if changed
+        if old_umask is not None:
+            os.umask(old_umask)
+        # Chown recursively, if UID or GID are specified
+        if ~UID or ~GID:
+            for root, dirs, files in os.walk(SRC_DIR):
+                for target in dirs + files:
+                    os.chown(os.path.join(root, target), UID, GID)
+
+
+def origin_for(folder):
+    """Guess the default git origin for that folder.
+
+    :param str folder:
+        Normally an absolute path to an expected git repo, whose name should
+        match the git repository where it comes from, using the env-supplied
+        pattern.
+    """
+    base = os.path.basename(folder)
+    pattern = DEFAULT_REPO_PATTERN
+    if base == "odoo":
+        pattern = DEFAULT_REPO_PATTERN_ODOO
+    return pattern.format(base)
+
+
+def missing_repos_config():
+    """Find the undefined repositories and return their default configuration.
+
+    :return dict:
+        git-aggregator-ready configuration dict for undefined repositories.
+    """
+    defined, expected = set(), {os.path.join(SRC_DIR, "odoo")}
+    # Find the repositories defined by hand
+    try:
+        with open(REPOS_YAML) as yaml_file:
+            for doc in yaml.load_all(yaml_file):
+                for repo in doc:
+                    defined.add(os.path.abspath(os.path.join(SRC_DIR, repo)))
+    except (IOError, AttributeError):
+        logger.debug("No repositories defined by hand")
+    # Find the repositories that should be present
+    try:
+        with open(ADDONS_YAML) as yaml_file:
+            for doc in yaml.load_all(yaml_file):
+                for repo in doc:
+                    if repo in {PRIVATE, CORE, "ONLY"}:
+                        continue
+                    repo_path = os.path.abspath(os.path.join(SRC_DIR, repo))
+                    if not os.path.exists(repo_path) or os.path.isdir(
+                            os.path.join(repo_path, ".git")):
+                        expected.add(repo_path)
+    except (IOError, AttributeError):
+        logger.debug("No addons are expected to be present")
+    # Find the undefined repositories and generate a config for them
+    missing = expected - defined
+    config = {
+        repo_path: {
+            'defaults': {'depth': '$DEPTH_DEFAULT'},
+            'merges': ['origin $ODOO_VERSION'],
+            'remotes': {
+                'origin': origin_for(repo_path),
+            },
+            'target': 'origin $ODOO_VERSION',
+        }
+        for repo_path in missing
+    }
+    logger.debug("Generated missing repos config %r", config)
+    return config
+
+
+# Aggregate user-specified repos
+if os.path.isfile(REPOS_YAML):
+    # HACK https://github.com/acsone/git-aggregator/pull/23
+    has_contents = True
+    with open(REPOS_YAML) as repos_file:
+        has_contents = yaml.load(repos_file)
+    if has_contents:
+        aggregate(REPOS_YAML)
+
+# Aggregate unspecified repos
+missing_config = missing_repos_config()
+if missing_config:
+    with open(AUTO_REPOS_YAML, "w") as autorepos:
+        yaml.dump(missing_config, autorepos)
+    aggregate(AUTO_REPOS_YAML)

lib/odoobaselib/__init__.py

@@ -10,13 +10,28 @@
 
 # Constants needed in scripts
 CUSTOM_DIR = "/opt/odoo/custom"
+AUTO_DIR = "/opt/odoo/auto"
+ADDONS_DIR = os.path.join(AUTO_DIR, 'addons')
 SRC_DIR = os.path.join(CUSTOM_DIR, 'src')
+
 ADDONS_YAML = os.path.join(SRC_DIR, 'addons')
 if os.path.isfile('%s.yaml' % ADDONS_YAML):
     ADDONS_YAML = '%s.yaml' % ADDONS_YAML
 else:
     ADDONS_YAML = '%s.yml' % ADDONS_YAML
-ADDONS_DIR = "/opt/odoo/auto/addons"
+
+REPOS_YAML = os.path.join(SRC_DIR, 'repos')
+if os.path.isfile('%s.yaml' % REPOS_YAML):
+    REPOS_YAML = '%s.yaml' % REPOS_YAML
+else:
+    REPOS_YAML = '%s.yml' % REPOS_YAML
+
+AUTO_REPOS_YAML = os.path.join(AUTO_DIR, 'repos')
+if os.path.isfile('%s.yml' % AUTO_REPOS_YAML):
+    AUTO_REPOS_YAML = '%s.yml' % AUTO_REPOS_YAML
+else:
+    AUTO_REPOS_YAML = '%s.yaml' % AUTO_REPOS_YAML
+
 CLEAN = os.environ.get("CLEAN") == "true"
 AUTO_REQUIREMENTS = os.environ.get("AUTO_REQUIREMENTS") == "true"
 LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR")
@@ -65,6 +80,9 @@ def addons_config(filtered=True, strict=False):
 
     :param bool strict:
         Use ``True`` to raise an exception if any declared addon is not found.
+
+    :return Iterator[str, str]:
+        A generator that yields ``(addon, repo)`` pairs.
     """
     config = dict()
     missing_glob = set()

tests/__init__.py

@@ -253,6 +253,8 @@ def test_dependencies(self):
                 ("test", "!", "-f", "custom/dependencies/gem.txt"),
                 ("test", "!", "-f", "custom/dependencies/npm.txt"),
                 ("test", "!", "-f", "custom/dependencies/pip.txt"),
+                # It should have module_auto_update available
+                ("test", "-d", "custom/src/server-tools/module_auto_update"),
                 # Patched Werkzeug version
                 ("bash", "-c", ('test "$(python -c "import werkzeug; '
                                 'print(werkzeug.__version__)")" == 0.14.1')),

tests/scaffoldings/dependencies/custom/src/addons.yaml

@@ -0,0 +1,2 @@
+server-tools:
+- module_auto_update