Support Ubuntu 22.04, jammy

.github/workflows/tests.yml

@@ -13,17 +13,18 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.10']
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install tox tox-gh-actions
-    - name: Test with tox
-      run: tox
+        python -m pip install .[test]
+    - name: Test with pytest
+      run: |
+        pytest

.gitignore

@@ -127,3 +127,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+.idea/
+

pyproject.toml

@@ -1,6 +1,3 @@
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
-
-[tool.black]
-target-version = ["py38"]

setup.cfg

@@ -30,6 +30,7 @@ install_requires =
     fabric < 2
     fabtools-python >= 0.19.7
     importlib_resources >= 1.4;python_version<'3.9'
+    setuptools;python_version>'3.11'
     Jinja2
     pycdstar >= 0.4.1
     pytz
@@ -63,6 +64,9 @@ minversion = 3.3
 testpaths = tests
 addopts = --cov=clldappconfig --cov-report=term-missing
 
+[coverage:report]
+skip_covered = True
+
 [flake8]
 # black compatible line lenght
 # cf. https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-length
@@ -72,24 +76,11 @@ extend-ignore = E126, E128, E203, E501, E731, W503
 exclude = .tox
 
 [tox:tox]
-envlist = py37, py38, py39, py310, linter
+envlist = py39, py310, py311, py312
 isolated_build = true
 skip_missing_interpreter = true
 
-[gh-actions]
-python =
-    3.7: py37
-    3.8: py38, linter
-    3.9: py39
-    3.10: py310
-
 [testenv]
 deps = .[test]
 commands = pytest {posargs}
 
-[testenv:linter]
-basepython = python3.8
-deps = .[dev]
-commands =
-    flake8 src/ tests/
-    black --check src/ tests/

src/clldappconfig/__init__.py

@@ -4,12 +4,11 @@
 
 from . import config
 
-__all__ = ["APPS_DIR", "CONFIG_FILE", "APPS", "init"]
+__all__ = ["SUPPORTED_LSB_RELEASES", "APPS_DIR", "CONFIG_FILE", "APPS", "init"]
 
+SUPPORTED_LSB_RELEASES = ['focal', 'jammy', 'noble']
 APPS_DIR = None
-
 CONFIG_FILE = None
-
 APPS = None
 
 

src/clldappconfig/commands/ls.py

@@ -20,7 +20,7 @@ def register(parser):
 
 def run(args):
     cols = {
-        "default": ["id", "url", "server", "port", "stack", "public"],
+        "default": ["id", "url", "server", "port", "public"],
         "administrative": ["id", "url", "editors", "contact"],
     }["administrative" if args.administrative else "default"]
     table = []
@@ -31,7 +31,6 @@ def run(args):
                 url="https://{0}".format(a.domain),
                 server=a.production,
                 port="{0}".format(a.port),
-                stack=a.stack,
                 editors=a.editors,
                 contact=a.contact,
                 public="{0}".format(a.public),
@@ -44,7 +43,6 @@ def run(args):
                     url="http://{0}/{1}".format(a.test, a.name),
                     server=a.test,
                     port="{0}".format(a.port),
-                    stack=a.stack,
                     editors="",
                     contact="",
                     public="{0}".format(False),

src/clldappconfig/config.py

@@ -108,7 +108,6 @@ class App(argparse.Namespace):
             "contact",
             "domain",
             "error_email",
-            "stack",
             "sqlalchemy_url",
             "app_pkg",
             "dbdump",
@@ -125,10 +124,10 @@ class App(argparse.Namespace):
             "workers": int,
             "timeout": int,
             "deploy_duration": int,
-            "require_deb_xenial": getwords,
-            "require_deb_bionic": getwords,
-            "require_deb_focal": getwords,
             "require_deb": getwords,
+            "require_deb_focal": getwords,
+            "require_deb_jammy": getwords,
+            "require_deb_noble": getwords,
             "require_pip": getwords,
             "pg_unaccent": getboolean,
         }
@@ -146,7 +145,6 @@ class App(argparse.Namespace):
                 "src_dir",
                 "static_dir",
                 "download_dir",
-                "alembic",
                 "gunicorn",
                 "log_dir",
                 "access_log",
@@ -157,7 +155,6 @@ class App(argparse.Namespace):
                 "nginx_site",
                 "nginx_location",
                 "nginx_htpasswd",
-                "varnish_site",
             ],
             pathlib.PurePosixPath,
         )

src/clldappconfig/fabtools/deb.py

@@ -0,0 +1,237 @@
+"""
+Debian packages
+===============
+
+This module provides tools to manage Debian/Ubuntu packages
+and repositories.
+
+"""
+from fabric.api import hide, run, settings
+
+from clldappconfig.fabtools.utils import run_as_root
+from clldappconfig.fabtools.files import getmtime, is_file
+
+MANAGER = 'DEBIAN_FRONTEND=noninteractive apt-get'
+
+
+def update_index(quiet=True):
+    """
+    Update APT package definitions.
+    """
+    options = "--quiet --quiet" if quiet else ""
+    run_as_root("%s %s update" % (MANAGER, options))
+
+
+def upgrade(safe=True):
+    """
+    Upgrade all packages.
+    """
+    manager = MANAGER
+    if safe:
+        cmd = 'upgrade'
+    else:
+        cmd = 'dist-upgrade'
+    run_as_root("%(manager)s --assume-yes %(cmd)s" % locals(), pty=False)
+
+
+def is_installed(pkg_name):
+    """
+    Check if a package is installed.
+    """
+    with settings(hide('running', 'stdout', 'stderr', 'warnings'), warn_only=True):
+        res = run("dpkg -s %(pkg_name)s" % locals())
+        for line in res.splitlines():
+            if line.startswith("Status: "):
+                status = line[8:]
+                if "installed" in status.split(' '):
+                    return True
+        return False
+
+
+def install(packages, update=False, options=None, version=None):
+    """
+    Install one or more packages.
+
+    If *update* is ``True``, the package definitions will be updated
+    first, using :py:func:`~fabtools.deb.update_index`.
+
+    Extra *options* may be passed to ``apt-get`` if necessary.
+
+    Example::
+
+        import fabtools
+
+        # Update index, then install a single package
+        fabtools.deb.install('build-essential', update=True)
+
+        # Install multiple packages
+        fabtools.deb.install([
+            'python-dev',
+            'libxml2-dev',
+        ])
+
+        # Install a specific version
+        fabtools.deb.install('emacs', version='23.3+1-1ubuntu9')
+
+    """
+    manager = MANAGER
+    if update:
+        update_index()
+    if options is None:
+        options = []
+    if version is None:
+        version = ''
+    if version and not isinstance(packages, list):
+        version = '=' + version
+    if not isinstance(packages, str):
+        packages = " ".join(packages)
+    options.append("--quiet")
+    options.append("--assume-yes")
+    options = " ".join(options)
+    cmd = '%(manager)s install %(options)s %(packages)s%(version)s' % locals()
+    run_as_root(cmd, pty=False)
+
+
+def uninstall(packages, purge=False, options=None):
+    """
+    Remove one or more packages.
+
+    If *purge* is ``True``, the package configuration files will be
+    removed from the system.
+
+    Extra *options* may be passed to ``apt-get`` if necessary.
+    """
+    manager = MANAGER
+    command = "purge" if purge else "remove"
+    if options is None:
+        options = []
+    if not isinstance(packages, str):
+        packages = " ".join(packages)
+    options.append("--assume-yes")
+    options = " ".join(options)
+    cmd = '%(manager)s %(command)s %(options)s %(packages)s' % locals()
+    run_as_root(cmd, pty=False)
+
+
+def preseed_package(pkg_name, preseed):
+    """
+    Enable unattended package installation by preseeding ``debconf``
+    parameters.
+
+    Example::
+
+        import fabtools
+
+        # Unattended install of Postfix mail server
+        fabtools.deb.preseed_package('postfix', {
+            'postfix/main_mailer_type': ('select', 'Internet Site'),
+            'postfix/mailname': ('string', 'example.com'),
+            'postfix/destinations': ('string', 'example.com, localhost.localdomain, localhost'),
+        })
+        fabtools.deb.install('postfix')
+
+    """
+    for q_name, _ in list(preseed.items()):
+        q_type, q_answer = _
+        run_as_root('echo "%(pkg_name)s %(q_name)s %(q_type)s %(q_answer)s" | debconf-set-selections' % locals())
+
+
+def get_selections():
+    """
+    Get the state of ``dkpg`` selections.
+
+    Returns a dict with state => [packages].
+    """
+    with settings(hide('stdout')):
+        res = run_as_root('dpkg --get-selections')
+    selections = dict()
+    for line in res.splitlines():
+        package, status = line.split()
+        selections.setdefault(status, list()).append(package)
+    return selections
+
+
+def apt_key_exists(keyid):
+    """
+    Check if the given key id exists in apt keyring.
+    """
+
+    # Command extracted from apt-key source
+    gpg_cmd = 'gpg --ignore-time-conflict --no-options --no-default-keyring --keyring /etc/apt/trusted.gpg'
+
+    with settings(hide('everything'), warn_only=True):
+        res = run('%(gpg_cmd)s --fingerprint %(keyid)s' % locals())
+
+    return res.succeeded
+
+
+def _check_pgp_key(path, keyid):
+    with settings(hide('everything')):
+        return not run('gpg --with-colons %(path)s | cut -d: -f 5 | grep -q \'%(keyid)s$\'' % locals())
+
+
+def add_apt_key(filename=None, url=None, keyid=None, keyserver='subkeys.pgp.net', update=False):
+    """
+    Trust packages signed with this public key.
+
+    Example::
+
+        import fabtools
+
+        # Varnish signing key from URL and verify fingerprint)
+        fabtools.deb.add_apt_key(keyid='C4DEFFEB', url='http://repo.varnish-cache.org/debian/GPG-key.txt')
+
+        # Nginx signing key from default key server (subkeys.pgp.net)
+        fabtools.deb.add_apt_key(keyid='7BD9BF62')
+
+        # From custom key server
+        fabtools.deb.add_apt_key(keyid='7BD9BF62', keyserver='keyserver.ubuntu.com')
+
+        # From a file
+        fabtools.deb.add_apt_key(keyid='7BD9BF62', filename='nginx.asc'
+    """
+
+    if keyid is None:
+        if filename is not None:
+            run_as_root('apt-key add %(filename)s' % locals())
+        elif url is not None:
+            run_as_root('wget %(url)s -O - | apt-key add -' % locals())
+        else:
+            raise ValueError('Either filename, url or keyid must be provided as argument')
+    else:
+        if filename is not None:
+            _check_pgp_key(filename, keyid)
+            run_as_root('apt-key add %(filename)s' % locals())
+        elif url is not None:
+            tmp_key = '/tmp/tmp.fabtools.key.%(keyid)s.key' % locals()
+            run_as_root('wget %(url)s -O %(tmp_key)s' % locals())
+            _check_pgp_key(tmp_key, keyid)
+            run_as_root('apt-key add %(tmp_key)s' % locals())
+        else:
+            keyserver_opt = '--keyserver %(keyserver)s' % locals() if keyserver is not None else ''
+            run_as_root('apt-key adv %(keyserver_opt)s --recv-keys %(keyid)s' % locals())
+
+    if update:
+        update_index()
+
+
+def last_update_time():
+    """
+    Get the time of last APT index update
+
+    This is the last modification time of ``/var/lib/apt/periodic/fabtools-update-success-stamp``.
+
+    Returns ``-1`` if there was no update before.
+
+    Example::
+
+        import fabtools
+
+        print(fabtools.deb.last_update_time())
+        # 1377603808.02
+
+    """
+    STAMP = '/var/lib/apt/periodic/fabtools-update-success-stamp'
+    if not is_file(STAMP):
+        return -1
+    return getmtime(STAMP)