diff --git a/11.0.Dockerfile b/11.0.Dockerfile index e753ab26..232b554b 100644 --- a/11.0.Dockerfile +++ b/11.0.Dockerfile @@ -135,6 +135,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"] ONBUILD CMD ["/usr/local/bin/odoo"] ONBUILD ARG AGGREGATE=true ONBUILD ARG AUTO_REQUIREMENTS=false +ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git" +ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git" ONBUILD ARG DEPTH_DEFAULT=1 ONBUILD ARG DEPTH_MERGE=100 ONBUILD ARG CLEAN=true @@ -156,6 +158,8 @@ ONBUILD ARG PGPORT=5432 ONBUILD ARG PGDATABASE=prod # Config variables ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \ + DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \ + DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \ UNACCENT="$UNACCENT" \ PGUSER="$PGUSER" \ PGPASSWORD="$PGPASSWORD" \ diff --git a/8.0.Dockerfile b/8.0.Dockerfile index aa6d5270..81d0dd2f 100644 --- a/8.0.Dockerfile +++ b/8.0.Dockerfile @@ -121,6 +121,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"] ONBUILD CMD ["/usr/local/bin/odoo"] ONBUILD ARG AGGREGATE=true ONBUILD ARG AUTO_REQUIREMENTS=false +ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git" +ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git" ONBUILD ARG DEPTH_DEFAULT=1 ONBUILD ARG DEPTH_MERGE=100 ONBUILD ARG CLEAN=true @@ -142,6 +144,8 @@ ONBUILD ARG PGPORT=5432 ONBUILD ARG PGDATABASE=prod # Config variables ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \ + DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \ + DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \ UNACCENT="$UNACCENT" \ PGUSER="$PGUSER" \ PGPASSWORD="$PGPASSWORD" \ diff --git a/bin/autoaggregate b/bin/autoaggregate index 46728ca6..ac0d0a7d 100755 --- a/bin/autoaggregate +++ b/bin/autoaggregate @@ -1,37 +1,130 @@ -#!/bin/bash -set -e - -conf=/opt/odoo/custom/src/repos - -if [ -f "${conf}.yaml" ]; then - conf="${conf}.yaml" -elif [ -f "${conf}.yml" ]; then - conf="${conf}.yml" -fi - -# Update linked repositories, if the `repos.yaml` file is found -if [ -f $conf ]; then - log INFO Aggregating repositories from $conf - cd $(dirname $conf) - - # Avoid wrong umask in aggregated files - if [ -n "$UMASK" ]; then - umask "$UMASK" - fi - - # Perform aggregation with environment variables expansion - set +e - gitaggregate --expand-env -c $conf - code=$? - set -e - - # Avoid wrong user/group in aggregated files - if [ -n "$GID" -a -n "$UID" ]; then - chown -R "$UID:$GID" . - fi - - [ $code -eq 0 ] || exit $code -else - log ERROR Cannot aggregate repositories: $conf not found - exit 1 -fi +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import sys +import yaml +from multiprocessing import cpu_count +from subprocess import check_call + +from odoobaselib import ( + ADDONS_YAML, + AUTO_REPOS_YAML, + CORE, + logger, + PRIVATE, + REPOS_YAML, + SRC_DIR, +) + +UMASK = os.environ.get("UMASK") +UID = int(os.environ.get("UID") or -1) +GID = int(os.environ.get("GID") or -1) +DEFAULT_REPO_PATTERN = os.environ.get("DEFAULT_REPO_PATTERN") +DEFAULT_REPO_PATTERN_ODOO = os.environ.get("DEFAULT_REPO_PATTERN_ODOO") + + +def aggregate(config): + """Execute git aggregator to pull git code. + + :param str config: + Path where to find the ``repos.yaml`` file. + """ + logger.info("Running gitaggregate with %s", config) + old_umask = None + try: + # Download git code with the specified umask, if any + if UMASK: + old_umask = os.umask(int(UMASK)) + check_call( + ["gitaggregate", "--expand-env", "--config", config, + "--jobs", str(cpu_count() or 1)], + cwd=SRC_DIR, + stderr=sys.stderr, + stdout=sys.stdout, + ) + finally: + # Restore umask, if changed + if old_umask is not None: + os.umask(old_umask) + # Chown recursively, if UID or GID are specified + if ~UID or ~GID: + for root, dirs, files in os.walk(SRC_DIR): + for target in dirs + files: + os.chown(os.path.join(root, target), UID, GID) + + +def origin_for(folder): + """Guess the default git origin for that folder. + + :param str folder: + Normally an absolute path to an expected git repo, whose name should + match the git repository where it comes from, using the env-supplied + pattern. + """ + base = os.path.basename(folder) + pattern = DEFAULT_REPO_PATTERN + if base == "odoo": + pattern = DEFAULT_REPO_PATTERN_ODOO + return pattern.format(base) + + +def missing_repos_config(): + """Find the undefined repositories and return their default configuration. + + :return dict: + git-aggregator-ready configuration dict for undefined repositories. + """ + defined, expected = set(), {os.path.join(SRC_DIR, "odoo")} + # Find the repositories defined by hand + try: + with open(REPOS_YAML) as yaml_file: + for doc in yaml.load_all(yaml_file): + for repo in doc: + defined.add(os.path.abspath(os.path.join(SRC_DIR, repo))) + except (IOError, AttributeError): + logger.debug("No repositories defined by hand") + # Find the repositories that should be present + try: + with open(ADDONS_YAML) as yaml_file: + for doc in yaml.load_all(yaml_file): + for repo in doc: + if repo in {PRIVATE, CORE, "ONLY"}: + continue + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + if not os.path.exists(repo_path) or os.path.isdir( + os.path.join(repo_path, ".git")): + expected.add(repo_path) + except (IOError, AttributeError): + logger.debug("No addons are expected to be present") + # Find the undefined repositories and generate a config for them + missing = expected - defined + config = { + repo_path: { + 'defaults': {'depth': '$DEPTH_DEFAULT'}, + 'merges': ['origin $ODOO_VERSION'], + 'remotes': { + 'origin': origin_for(repo_path), + }, + 'target': 'origin $ODOO_VERSION', + } + for repo_path in missing + } + logger.debug("Generated missing repos config %r", config) + return config + + +# Aggregate user-specified repos +if os.path.isfile(REPOS_YAML): + # HACK https://github.com/acsone/git-aggregator/pull/23 + has_contents = True + with open(REPOS_YAML) as repos_file: + has_contents = yaml.load(repos_file) + if has_contents: + aggregate(REPOS_YAML) + +# Aggregate unspecified repos +missing_config = missing_repos_config() +if missing_config: + with open(AUTO_REPOS_YAML, "w") as autorepos: + yaml.dump(missing_config, autorepos) + aggregate(AUTO_REPOS_YAML) diff --git a/lib/odoobaselib/__init__.py b/lib/odoobaselib/__init__.py index 7b60cabe..1e650ea2 100644 --- a/lib/odoobaselib/__init__.py +++ b/lib/odoobaselib/__init__.py @@ -10,13 +10,28 @@ # Constants needed in scripts CUSTOM_DIR = "/opt/odoo/custom" +AUTO_DIR = "/opt/odoo/auto" +ADDONS_DIR = os.path.join(AUTO_DIR, 'addons') SRC_DIR = os.path.join(CUSTOM_DIR, 'src') + ADDONS_YAML = os.path.join(SRC_DIR, 'addons') if os.path.isfile('%s.yaml' % ADDONS_YAML): ADDONS_YAML = '%s.yaml' % ADDONS_YAML else: ADDONS_YAML = '%s.yml' % ADDONS_YAML -ADDONS_DIR = "/opt/odoo/auto/addons" + +REPOS_YAML = os.path.join(SRC_DIR, 'repos') +if os.path.isfile('%s.yaml' % REPOS_YAML): + REPOS_YAML = '%s.yaml' % REPOS_YAML +else: + REPOS_YAML = '%s.yml' % REPOS_YAML + +AUTO_REPOS_YAML = os.path.join(AUTO_DIR, 'repos') +if os.path.isfile('%s.yml' % AUTO_REPOS_YAML): + AUTO_REPOS_YAML = '%s.yml' % AUTO_REPOS_YAML +else: + AUTO_REPOS_YAML = '%s.yaml' % AUTO_REPOS_YAML + CLEAN = os.environ.get("CLEAN") == "true" AUTO_REQUIREMENTS = os.environ.get("AUTO_REQUIREMENTS") == "true" LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR") @@ -65,6 +80,9 @@ def addons_config(filtered=True, strict=False): :param bool strict: Use ``True`` to raise an exception if any declared addon is not found. + + :return Iterator[str, str]: + A generator that yields ``(addon, repo)`` pairs. """ config = dict() missing_glob = set() diff --git a/tests/__init__.py b/tests/__init__.py index 97bf23c0..8c0bf09e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -253,6 +253,8 @@ def test_dependencies(self): ("test", "!", "-f", "custom/dependencies/gem.txt"), ("test", "!", "-f", "custom/dependencies/npm.txt"), ("test", "!", "-f", "custom/dependencies/pip.txt"), + # It should have module_auto_update available + ("test", "-d", "custom/src/server-tools/module_auto_update"), # Patched Werkzeug version ("bash", "-c", ('test "$(python -c "import werkzeug; ' 'print(werkzeug.__version__)")" == 0.14.1')), diff --git a/tests/scaffoldings/dependencies/custom/src/addons.yaml b/tests/scaffoldings/dependencies/custom/src/addons.yaml index e69de29b..c6df2c52 100644 --- a/tests/scaffoldings/dependencies/custom/src/addons.yaml +++ b/tests/scaffoldings/dependencies/custom/src/addons.yaml @@ -0,0 +1,2 @@ +server-tools: +- module_auto_update diff --git a/tests/scaffoldings/smallest/custom/src/repos.yaml b/tests/scaffoldings/smallest/custom/src/repos.yaml deleted file mode 100644 index 1eddc934..00000000 --- a/tests/scaffoldings/smallest/custom/src/repos.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Odoo is always required -./odoo: - defaults: - # Shallow repositores are faster & thinner - depth: $DEPTH_DEFAULT - remotes: - ocb: https://github.com/OCA/OCB.git - odoo: https://github.com/odoo/odoo.git - target: - ocb $ODOO_VERSION - merges: - - ocb $ODOO_VERSION