Merge pull request #13 from vimc/VIMC-7467

Add support for certificates from Let's Encrypt.

Author: plietar

.github/workflows/test.yml

@@ -49,12 +49,12 @@ jobs:
           docker pull vimc/montagu-db:master
           docker pull vimc/montagu-migrate:master
           docker pull vimc/montagu-reverse-proxy:master
-          docker pull vimc/montagu-reverse-proxy:vimc-7152
           docker pull vimc/orderly-web-user-cli:master
           docker pull vimc/orderly-web:master
           docker pull vimc/orderly.server:master
           docker pull vimc/orderlyweb-migrate:master
           docker pull vimc/task-queue-worker:master
+          docker pull ghcr.io/letsencrypt/pebble:latest
       - name: Test
         env:
           VAULT_TOKEN: ${{ secrets.VAULT_TOKEN }}

README.md

@@ -16,28 +16,7 @@ pip install montagu-deploy
 ## Usage
 
 ```
-$ montagu --help
-Usage:
-  montagu --version
-  montagu start <path> [--extra=PATH] [--option=OPTION]... [--pull]
-  montagu status <path>
-  montagu stop <path> [--volumes] [--network] [--kill] [--force]
-    [--extra=PATH] [--option=OPTION]...
-
-Options:
-  --extra=PATH     Path, relative to <path>, of yml file of additional
-                   configuration
-  --option=OPTION  Additional configuration options, in the form key=value
-                   Use dots in key for hierarchical structure, e.g., a.b=value
-                   This argument may be repeated to provide multiple arguments
-  --pull           Pull images before starting
-  --volumes        Remove volumes (WARNING: irreversible data loss)
-  --network        Remove network
-  --kill           Kill the containers (faster, but possible db corruption)
-  --force          Force stop even if containers are corrupted and cannot
-                   signal their running configuration, or if config cannot be
-                   parsed. Use with extra and/or option to force stop with
-                   configuration options.
+$ montagu start <path>
 ```
 
 Here `<path>` is the path to a directory that contains a configuration file `montagu.yml`.

config/acme/diagnostic-reports.yml

@@ -0,0 +1,9 @@
+testGroup:
+  testDisease:
+    - report_name: diagnostic
+      assignee: a.hill
+      success_email:
+        recipients:
+          - [email protected]
+          - [email protected]
+        subject: "VIMC diagnostic report: {touchstone} - {group} - {disease}"

config/acme/montagu.yml

@@ -0,0 +1,125 @@
+## Prefix for container names; we'll use {container_prefix}-(container_name)
+container_prefix: montagu
+
+## Set this flag to true to prevent use of --volumes in the cli to remove
+## volumes on stop
+protect_data: false
+
+## Docker org for images
+repo: vimc
+
+## The name of the docker network that containers will be attached to.
+## If you want to proxy Packit to the host, you will need to
+## arrange a proxy on this network
+network: montagu-network
+
+# Domain where this instance of Montagu will be deployed. E.g. science.montagu.dide.ic.uk
+hostname: montagu.org
+
+## Names of the docker volumes to use
+volumes:
+  db: db_volume
+  burden_estimates: burden_estimate_files
+  emails: emails
+  templates: template_volume
+  guidance: guidance_volume
+  mq: mq
+  acme-challenge: acme-challenge
+  certificates: certificates
+  certbot: certbot
+
+api:
+  name: montagu-api
+  tag: master
+  admin:
+    name: montagu-cli
+    tag: master
+db:
+  name: montagu-db
+  tag: master
+  root_user: vimc
+  migrate:
+    name: montagu-migrate
+    tag: master
+  users:
+    api:
+      password: "apipassword"
+      permissions: all
+    import:
+      password: "importpassword"
+      permissions: all
+    orderly:
+      password: "orderlypassword"
+      permissions: all
+    readonly:
+      password: "readonlypassword"
+      permissions: readonly
+  protected_tables:
+    - gavi_support_level
+    - activity_type
+    - burden_outcome
+    - gender
+    - responsibility_set_status
+    - impact_outcome
+    - gavi_support_level
+    - support_type
+    - touchstone_status
+    - permission
+    - role
+    - role_permission
+proxy:
+  name: montagu-reverse-proxy
+  tag: master
+  port_http: 80
+  port_https: 443
+  metrics:
+    repo: nginx
+    name: nginx-prometheus-exporter
+    tag: 1.3.0
+  acme:
+    email: [email protected]
+    additional_domains:
+      - montagu-dev.org
+contrib:
+  name: montagu-contrib-portal
+  tag: master
+admin:
+  name: montagu-admin-portal
+  tag: master
+mq:
+  repo: docker.io
+  name: redis
+  tag: latest
+  port: 6379
+flower:
+  repo: mher
+  name: flower
+  tag: 0.9.5
+  port: 5555
+task_queue:
+  name: task-queue-worker
+  tag: master
+  tasks:
+    diagnostic_reports:
+      use_additional_recipients: false
+      poll_seconds: 5
+    archive_folder_contents:
+      min_file_age_seconds: 3600
+  servers:
+    youtrack:
+      token: faketoken
+    orderlyweb:
+      url: http://orderly-web-web:8888
+    montagu:
+      user: [email protected]
+      password: password
+    smtp:
+      from: [email protected]
+# If fake_smtp_server config is provided, the task_queue will use this as its smtp server
+# Note this will override other config provided in the task_queue section above
+fake_smtp_server:
+  repo: reachfive
+  name: fake-smtp-server
+  tag: latest
+
+orderly_web_api_url: https://localhost/reports/api/v2

config/basic/montagu.yml

@@ -66,7 +66,7 @@ db:
     - role_permission
 proxy:
   name: montagu-reverse-proxy
-  tag: vimc-7152
+  tag: master
   port_http: 80
   port_https: 443
   metrics:

config/ci/montagu.yml

@@ -83,7 +83,7 @@ db:
     - role_permission
 proxy:
   name: montagu-reverse-proxy
-  tag: vimc-7152
+  tag: master
   port_http: 80
   port_https: 443
   metrics:

config/complete/montagu.yml

@@ -94,7 +94,6 @@ proxy:
   ssl:
     key: "k3y"
     certificate: "cert"
-    dhparam: "param"
 contrib:
   name: montagu-contrib-portal
   tag: master

pyproject.toml

@@ -49,7 +49,8 @@ dependencies = [
   "pytest",
   "redis",
   "vault_dev",
-  "YTClient"
+  "YTClient",
+  "cryptography"
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"

src/montagu_deploy/certbot.py

@@ -0,0 +1,90 @@
+# https://github.com/certbot/certbot/blob/v3.0.1/acme/examples/http01_example.py
+
+import os.path
+import sys
+import tarfile
+from tempfile import TemporaryFile
+
+import docker
+from constellation import docker_util
+
+# The Docker API uses Go's FileMode values. These are different from the
+# standard values, as found in eg. stat.S_IFLNK.
+# https://pkg.go.dev/io/fs#FileMode
+DOCKER_MODE_TYPE = 0x8F280000
+DOCKER_MODE_SYMLINK = 0x8000000
+
+
+def read_file(container, path, *, follow_links=False):
+    stream, status = container.get_archive(path)
+    if follow_links and (status["mode"] & DOCKER_MODE_TYPE) == DOCKER_MODE_SYMLINK:
+        return read_file(container, status["linkTarget"], follow_links=False)
+    else:
+        with TemporaryFile() as f:
+            for d in stream:
+                f.write(d)
+            f.seek(0)
+
+            with tarfile.open(fileobj=f) as tar:
+                return tar.extractfile(os.path.basename(path)).read()
+
+
+def obtain_certificate(cfg, extra_args):
+    docker_util.ensure_volume(cfg.volumes["certbot"])
+    docker_util.ensure_volume(cfg.volumes["acme-challenge"])
+
+    environment = {}
+    command = [
+        "certonly",
+        "--non-interactive",
+        "--agree-tos",
+        "--webroot",
+        "--webroot-path=/var/www",
+        f"--email={cfg.acme_email}",
+        f"--domain={cfg.hostname}",
+    ]
+
+    for d in cfg.acme_additional_domains:
+        command.append(f"--domain={d}")
+
+    if cfg.acme_server:
+        command.append(f"--server={cfg.acme_server}"),
+    if cfg.acme_no_verify_ssl:
+        command.append("--no-verify-ssl")
+        environment["PYTHONWARNINGS"] = "ignore:Unverified HTTPS request"
+
+    command.extend(extra_args)
+
+    image = "certbot/certbot"
+    container = docker.from_env().containers.run(
+        image,
+        command=command,
+        detach=True,
+        volumes={
+            cfg.volumes["acme-challenge"]: {
+                "bind": "/var/www/.well-known/acme-challenge",
+                "mode": "rw",
+            },
+            cfg.volumes["certbot"]: {
+                "bind": "/etc/letsencrypt",
+                "mode": "rw",
+            },
+        },
+        network=cfg.network,
+        environment=environment,
+    )
+
+    try:
+        exit_status = container.wait()["StatusCode"]
+
+        sys.stderr.write(container.logs().decode("utf-8"))
+        if exit_status != 0:
+            raise docker.errors.ContainerError(container, exit_status, command, image, None)
+
+        cert = read_file(container, f"/etc/letsencrypt/live/{cfg.hostname}/fullchain.pem", follow_links=True)
+        key = read_file(container, f"/etc/letsencrypt/live/{cfg.hostname}/privkey.pem", follow_links=True)
+
+        return (cert, key)
+
+    finally:
+        container.remove()

src/montagu_deploy/cli.py

@@ -4,6 +4,7 @@
   montagu status <path>
   montagu stop <path> [--volumes] [--network] [--kill] [--force]
     [--extra=PATH] [--option=OPTION]...
+  montagu renew-certificate <path> [--option=OPTION]... [--] [ARGS...]
 
 Options:
   --extra=PATH     Path, relative to <path>, of yml file of additional
@@ -15,24 +16,21 @@
   --volumes        Remove volumes (WARNING: irreversible data loss)
   --network        Remove network
   --kill           Kill the containers (faster, but possible db corruption)
-  --force          Force stop even if containers are corrupted and cannot
-                   signal their running configuration, or if config cannot be
-                   parsed. Use with extra and/or option to force stop with
-                   configuration options.
 """
 
 import docopt
 import yaml
 
 import montagu_deploy.__about__ as about
+from montagu_deploy.certbot import obtain_certificate
 from montagu_deploy.config import MontaguConfig
-from montagu_deploy.montagu_constellation import montagu_constellation
+from montagu_deploy.montagu_constellation import montagu_constellation, proxy_update_certificate
 
 
 def main(argv=None):
     path, extra, options, args = parse_args(argv)
     if args.version:
-        return about.__version__
+        print(about.__version__)
     else:
         cfg = MontaguConfig(path, extra, options)
         obj = montagu_constellation(cfg)
@@ -42,7 +40,8 @@ def main(argv=None):
             montagu_status(obj)
         elif args.action == "stop":
             montagu_stop(obj, args, cfg)
-        return True
+        elif args.action == "renew-certificate":
+            montagu_renew_certificate(obj, cfg, args.extra_args)
 
 
 def parse_args(argv=None):
@@ -61,6 +60,18 @@ def montagu_status(obj):
     obj.status()
 
 
+def montagu_renew_certificate(obj, cfg, extra_args):
+    if cfg.ssl_mode != "acme":
+        msg = "Proxy is not configured to use automatic certificates"
+        raise Exception(msg)
+
+    print("Renewing certificates")
+    (cert, key) = obtain_certificate(cfg, extra_args)
+
+    container = obj.containers.get("proxy", cfg.container_prefix)
+    proxy_update_certificate(container, cert, key, reload=True)
+
+
 def montagu_stop(obj, args, cfg):
     if args.volumes:
         verify_data_loss(cfg)
@@ -123,9 +134,12 @@ def __init__(self, args):
             self.action = "status"
         elif args["stop"]:
             self.action = "stop"
+        elif args["renew-certificate"]:
+            self.action = "renew-certificate"
 
         self.pull = args["--pull"]
         self.kill = args["--kill"]
         self.volumes = args["--volumes"]
         self.network = args["--network"]
         self.version = args["--version"]
+        self.extra_args = args["ARGS"]