From 070323c871586ef2fad8998ee9c6a56d51ec9332 Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Thu, 13 Sep 2018 12:38:59 +0100 Subject: [PATCH] Fix automatic repos.yaml generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this change, definitely we can have a Doodba scaffolding without `repos.yaml` file. It will do this: 1. Download all git code from repos in `repos.yaml`, if any. 2. Get the list of expected code repos from `addons.yaml`. 3. Autogenerate a `repos.yaml` file for all expected but absent repos, based on the provided patterns. 4. Download all of that missing code. 5. Continue normal operation. As a 🎁, we now download git code in parallel if the building machine has more than 1 CPU. Some tests have been modified to ensure they still pass with this new feature. --- 11.0.Dockerfile | 4 +- 8.0.Dockerfile | 4 +- bin/autoaggregate | 167 ++++++++++++++---- build.d/300-oca-dependencies | 97 ---------- lib/odoobaselib/__init__.py | 15 +- tests/__init__.py | 2 + .../dependencies/custom/src/addons.yaml | 2 + .../smallest/custom/src/repos.yaml | 12 -- 8 files changed, 146 insertions(+), 157 deletions(-) delete mode 100755 build.d/300-oca-dependencies delete mode 100644 tests/scaffoldings/smallest/custom/src/repos.yaml diff --git a/11.0.Dockerfile b/11.0.Dockerfile index c5088acb..232b554b 100644 --- a/11.0.Dockerfile +++ b/11.0.Dockerfile @@ -135,7 +135,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"] ONBUILD CMD ["/usr/local/bin/odoo"] ONBUILD ARG AGGREGATE=true ONBUILD ARG AUTO_REQUIREMENTS=false -ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/%s.git" +ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git" +ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git" ONBUILD ARG DEPTH_DEFAULT=1 ONBUILD ARG DEPTH_MERGE=100 ONBUILD ARG CLEAN=true @@ -158,6 +159,7 @@ ONBUILD ARG PGDATABASE=prod # Config variables ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \ DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \ + DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \ UNACCENT="$UNACCENT" \ PGUSER="$PGUSER" \ PGPASSWORD="$PGPASSWORD" \ diff --git a/8.0.Dockerfile b/8.0.Dockerfile index ac5f359d..81d0dd2f 100644 --- a/8.0.Dockerfile +++ b/8.0.Dockerfile @@ -121,7 +121,8 @@ ONBUILD ENTRYPOINT ["/opt/odoo/common/entrypoint"] ONBUILD CMD ["/usr/local/bin/odoo"] ONBUILD ARG AGGREGATE=true ONBUILD ARG AUTO_REQUIREMENTS=false -ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/%s.git" +ONBUILD ARG DEFAULT_REPO_PATTERN="https://github.com/OCA/{}.git" +ONBUILD ARG DEFAULT_REPO_PATTERN_ODOO="https://github.com/OCA/OCB.git" ONBUILD ARG DEPTH_DEFAULT=1 ONBUILD ARG DEPTH_MERGE=100 ONBUILD ARG CLEAN=true @@ -144,6 +145,7 @@ ONBUILD ARG PGDATABASE=prod # Config variables ONBUILD ENV ADMIN_PASSWORD="$ADMIN_PASSWORD" \ DEFAULT_REPO_PATTERN="$DEFAULT_REPO_PATTERN" \ + DEFAULT_REPO_PATTERN_ODOO="$DEFAULT_REPO_PATTERN_ODOO" \ UNACCENT="$UNACCENT" \ PGUSER="$PGUSER" \ PGPASSWORD="$PGPASSWORD" \ diff --git a/bin/autoaggregate b/bin/autoaggregate index 46728ca6..ac0d0a7d 100755 --- a/bin/autoaggregate +++ b/bin/autoaggregate @@ -1,37 +1,130 @@ -#!/bin/bash -set -e - -conf=/opt/odoo/custom/src/repos - -if [ -f "${conf}.yaml" ]; then - conf="${conf}.yaml" -elif [ -f "${conf}.yml" ]; then - conf="${conf}.yml" -fi - -# Update linked repositories, if the `repos.yaml` file is found -if [ -f $conf ]; then - log INFO Aggregating repositories from $conf - cd $(dirname $conf) - - # Avoid wrong umask in aggregated files - if [ -n "$UMASK" ]; then - umask "$UMASK" - fi - - # Perform aggregation with environment variables expansion - set +e - gitaggregate --expand-env -c $conf - code=$? - set -e - - # Avoid wrong user/group in aggregated files - if [ -n "$GID" -a -n "$UID" ]; then - chown -R "$UID:$GID" . - fi - - [ $code -eq 0 ] || exit $code -else - log ERROR Cannot aggregate repositories: $conf not found - exit 1 -fi +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import sys +import yaml +from multiprocessing import cpu_count +from subprocess import check_call + +from odoobaselib import ( + ADDONS_YAML, + AUTO_REPOS_YAML, + CORE, + logger, + PRIVATE, + REPOS_YAML, + SRC_DIR, +) + +UMASK = os.environ.get("UMASK") +UID = int(os.environ.get("UID") or -1) +GID = int(os.environ.get("GID") or -1) +DEFAULT_REPO_PATTERN = os.environ.get("DEFAULT_REPO_PATTERN") +DEFAULT_REPO_PATTERN_ODOO = os.environ.get("DEFAULT_REPO_PATTERN_ODOO") + + +def aggregate(config): + """Execute git aggregator to pull git code. + + :param str config: + Path where to find the ``repos.yaml`` file. + """ + logger.info("Running gitaggregate with %s", config) + old_umask = None + try: + # Download git code with the specified umask, if any + if UMASK: + old_umask = os.umask(int(UMASK)) + check_call( + ["gitaggregate", "--expand-env", "--config", config, + "--jobs", str(cpu_count() or 1)], + cwd=SRC_DIR, + stderr=sys.stderr, + stdout=sys.stdout, + ) + finally: + # Restore umask, if changed + if old_umask is not None: + os.umask(old_umask) + # Chown recursively, if UID or GID are specified + if ~UID or ~GID: + for root, dirs, files in os.walk(SRC_DIR): + for target in dirs + files: + os.chown(os.path.join(root, target), UID, GID) + + +def origin_for(folder): + """Guess the default git origin for that folder. + + :param str folder: + Normally an absolute path to an expected git repo, whose name should + match the git repository where it comes from, using the env-supplied + pattern. + """ + base = os.path.basename(folder) + pattern = DEFAULT_REPO_PATTERN + if base == "odoo": + pattern = DEFAULT_REPO_PATTERN_ODOO + return pattern.format(base) + + +def missing_repos_config(): + """Find the undefined repositories and return their default configuration. + + :return dict: + git-aggregator-ready configuration dict for undefined repositories. + """ + defined, expected = set(), {os.path.join(SRC_DIR, "odoo")} + # Find the repositories defined by hand + try: + with open(REPOS_YAML) as yaml_file: + for doc in yaml.load_all(yaml_file): + for repo in doc: + defined.add(os.path.abspath(os.path.join(SRC_DIR, repo))) + except (IOError, AttributeError): + logger.debug("No repositories defined by hand") + # Find the repositories that should be present + try: + with open(ADDONS_YAML) as yaml_file: + for doc in yaml.load_all(yaml_file): + for repo in doc: + if repo in {PRIVATE, CORE, "ONLY"}: + continue + repo_path = os.path.abspath(os.path.join(SRC_DIR, repo)) + if not os.path.exists(repo_path) or os.path.isdir( + os.path.join(repo_path, ".git")): + expected.add(repo_path) + except (IOError, AttributeError): + logger.debug("No addons are expected to be present") + # Find the undefined repositories and generate a config for them + missing = expected - defined + config = { + repo_path: { + 'defaults': {'depth': '$DEPTH_DEFAULT'}, + 'merges': ['origin $ODOO_VERSION'], + 'remotes': { + 'origin': origin_for(repo_path), + }, + 'target': 'origin $ODOO_VERSION', + } + for repo_path in missing + } + logger.debug("Generated missing repos config %r", config) + return config + + +# Aggregate user-specified repos +if os.path.isfile(REPOS_YAML): + # HACK https://github.com/acsone/git-aggregator/pull/23 + has_contents = True + with open(REPOS_YAML) as repos_file: + has_contents = yaml.load(repos_file) + if has_contents: + aggregate(REPOS_YAML) + +# Aggregate unspecified repos +missing_config = missing_repos_config() +if missing_config: + with open(AUTO_REPOS_YAML, "w") as autorepos: + yaml.dump(missing_config, autorepos) + aggregate(AUTO_REPOS_YAML) diff --git a/build.d/300-oca-dependencies b/build.d/300-oca-dependencies deleted file mode 100755 index 92b1f7a4..00000000 --- a/build.d/300-oca-dependencies +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python - -import os -import sys -import yaml -import logging -import urllib2 -from glob import iglob - -from odoobaselib import ( - CUSTOM_DIR, AUTO_ADDONS_YAML, AUTO_REPOS_YAML, ADDONS_YAML, REPOS_YAML -) - -# Skip aggregation of dependencies if AGGREGATE env var is not set -if not os.environ.get('AGGREGATE'): - logging.warning("Skipping oca dependency aggregation") - sys.exit() - -BASE_URL = 'https://raw.githubusercontent.com/OCA' - - -# Method from maintainer-quality-tools -def parse_depfile(depfile, owner='OCA'): - deps = set() - for line in depfile: - line = line.strip() - if not line or line.startswith('#'): - continue - parts = line.split() - repo = parts[0] - if len(parts) > 2: - branch = parts[2] - else: - branch = os.environ['ODOO_VERSION'] - if len(parts) > 1: - url = parts[1] - else: - url = os.environ['DEFAULT_REPO_PATTERN'] % repo - deps.add((repo, url, branch)) - return deps - - -def process_dependencies(dep_list): - """Attempt fetching oca_dependencies.txt files from github to build a list - of dependencies in order to aggregate only once""" - processed_dependencies = set() - for dep in dep_list: - if dep in processed_dependencies: - continue - oca_dep_url = '%s/%s/%s/oca_dependencies.txt' % ( - BASE_URL, dep[0], branch - ) - logging.info('Attempting to fetch %s' % oca_dep_url) - try: - data = urllib2.urlopen(oca_dep_url) - dependencies = set(parse_depfile(data)) - if dep[0] not in ADDONS_YAML.keys(): - processed_dependencies.add(dep) - processed_dependencies |= process_dependencies(dependencies) - logging.info('Fetched dependencies for %s' % oca_dep_url) - except Exception as e: - # Not neccesairly error but it might not have oca_dependencies.txt - logging.warning('Error fetching url "%s". Error: %s' % ( - oca_dep_url, e.code) - ) - # TODO: Retry mechanism - return processed_dependencies - -branch = os.environ['ODOO_VERSION'] - -# Find all dependency files after initial aggregation -dep_files = iglob('%s/*/oca_dependencies.txt' % CUSTOM_DIR) -initial_dependencies = set() - -# Get all initial repositories across all projects -for dep in dep_files: - file_obj = open(dep, 'r') - deps = set(parse_depfile(file_obj)) - initial_dependencies.update(deps) - -# Get all subsequent repositories from oca_dependencies via github -dependencies = process_dependencies(initial_dependencies) - -# Generate auto_addons and auto_repos yaml files -for dep in dependencies: - ADDONS_YAML[dep[0]] = ['*'] - - repo_vals = { - 'defaults': {'depth': '$DEPTH_DEFAULT'}, - 'remotes': {'origin': dep[1]}, - 'target': {'origin': '$ODOO_VERSION'}, - 'merges': [{'origin': '$ODOO_VERSION'}], - } - REPOS_YAML[dep[0]] = {'%s' % dep[0]: repo_vals} - -yaml.dump(ADDONS_YAML, AUTO_ADDONS_YAML, default_flow_style=False) -yaml.dump(REPOS_YAML, AUTO_REPOS_YAML, default_flow_style=False) diff --git a/lib/odoobaselib/__init__.py b/lib/odoobaselib/__init__.py index 590ade77..1e650ea2 100644 --- a/lib/odoobaselib/__init__.py +++ b/lib/odoobaselib/__init__.py @@ -26,17 +26,11 @@ else: REPOS_YAML = '%s.yml' % REPOS_YAML -AUTO_ADDONS_YAML = os.path.join(AUTO_DIR, 'addons') -if os.path.isfile('%s.yaml' % AUTO_ADDONS_YAML): - AUTO_ADDONS_YAML = '%s.yaml' % AUTO_ADDONS_YAML -else: - AUTO_ADDONS_YAML = '%s.yml' % AUTO_ADDONS_YAML - AUTO_REPOS_YAML = os.path.join(AUTO_DIR, 'repos') -if os.path.isfile('%s.yaml' % AUTO_REPOS_YAML): - AUTO_REPOS_YAML = '%s.yaml' % AUTO_REPOS_YAML -else: +if os.path.isfile('%s.yml' % AUTO_REPOS_YAML): AUTO_REPOS_YAML = '%s.yml' % AUTO_REPOS_YAML +else: + AUTO_REPOS_YAML = '%s.yaml' % AUTO_REPOS_YAML CLEAN = os.environ.get("CLEAN") == "true" AUTO_REQUIREMENTS = os.environ.get("AUTO_REQUIREMENTS") == "true" @@ -86,6 +80,9 @@ def addons_config(filtered=True, strict=False): :param bool strict: Use ``True`` to raise an exception if any declared addon is not found. + + :return Iterator[str, str]: + A generator that yields ``(addon, repo)`` pairs. """ config = dict() missing_glob = set() diff --git a/tests/__init__.py b/tests/__init__.py index 97bf23c0..8c0bf09e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -253,6 +253,8 @@ def test_dependencies(self): ("test", "!", "-f", "custom/dependencies/gem.txt"), ("test", "!", "-f", "custom/dependencies/npm.txt"), ("test", "!", "-f", "custom/dependencies/pip.txt"), + # It should have module_auto_update available + ("test", "-d", "custom/src/server-tools/module_auto_update"), # Patched Werkzeug version ("bash", "-c", ('test "$(python -c "import werkzeug; ' 'print(werkzeug.__version__)")" == 0.14.1')), diff --git a/tests/scaffoldings/dependencies/custom/src/addons.yaml b/tests/scaffoldings/dependencies/custom/src/addons.yaml index e69de29b..c6df2c52 100644 --- a/tests/scaffoldings/dependencies/custom/src/addons.yaml +++ b/tests/scaffoldings/dependencies/custom/src/addons.yaml @@ -0,0 +1,2 @@ +server-tools: +- module_auto_update diff --git a/tests/scaffoldings/smallest/custom/src/repos.yaml b/tests/scaffoldings/smallest/custom/src/repos.yaml deleted file mode 100644 index 1eddc934..00000000 --- a/tests/scaffoldings/smallest/custom/src/repos.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Odoo is always required -./odoo: - defaults: - # Shallow repositores are faster & thinner - depth: $DEPTH_DEFAULT - remotes: - ocb: https://github.com/OCA/OCB.git - odoo: https://github.com/odoo/odoo.git - target: - ocb $ODOO_VERSION - merges: - - ocb $ODOO_VERSION