.github/workflows/build-workflow.yaml

@@ -10,10 +10,12 @@ on:
   create:
   # Run on pull requests
   pull_request:
+  # Allow this workflow to be called  by others
+  workflow_call:
 
 env:
-  elixir_version: "1.14.5"
-  otp_version: "25.3.2"
+  elixir_version: "1.15.7"
+  otp_version: "26.1"
 
 jobs:
   test-dialyzer:
@@ -33,7 +35,8 @@ jobs:
         key: ${{ runner.os }}-${{ env.elixir_version }}-${{ env.otp_version }}-_build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}-${{ github.sha }}
         restore-keys: |
           ${{ runner.os }}-${{ env.elixir_version }}-${{ env.otp_version }}-_build-${{ hashFiles(format('{0}{1}', github.workspace, '/mix.lock')) }}
-    - uses: actions/cache@v1
+    - id: plt_cache
+      uses: actions/cache@v1
       with:
         path: dialyzer_cache
         key: ${{ runner.os }}-${{ env.elixir_version }}-${{ env.otp_version }}-dialyzer_cache-${{ github.sha }}
@@ -45,6 +48,9 @@ jobs:
         elixir-version: ${{ env.elixir_version }}
     - name: Install Dependencies
       run: mix deps.get
+    - name: Create PLTs dir
+      if: ${{steps.plt_cache.outputs.cache-hit != 'true'}}
+      run: mkdir -p dialyzer_cache && mix dialyzer --plt
     - name: Run dialyzer
       # FIXME: This should be set to fail when dialyzer issues are fixed
       run: mix dialyzer || exit 0
@@ -60,15 +66,23 @@ jobs:
       matrix:
         rabbitmq:
         - "rabbitmq:3.12.0-management"
+        database:
+        - "cassandra:3.11.15"
+        - "scylladb/scylla:5.2.2"
     services:
       rabbitmq:
         image: ${{ matrix.rabbitmq }}
         ports:
         - 5672:5672
         - 15672:15672
+      database:
+        image: ${{ matrix.database }}
+        ports:
+        - 9042:9042
     env:
       MIX_ENV: test
       RABBITMQ_HOST: localhost
+      CASSANDRA_NODES: localhost
     steps:
     - uses: actions/checkout@v2
     - uses: actions/cache@v1
@@ -93,6 +107,10 @@ jobs:
       run: mix format --check-formatted
     - name: Compile
       run: mix do compile
+    - name: Wait for Cassandra
+      run: |
+        npm install -g wait-for-cassandra
+        wait-for-cassandra -T 120000 -h $CASSANDRA_NODES
     - name: Test and Coverage
       run: mix coveralls.json  --exclude wip -o coverage_results
     - name: Upload Coverage Results to CodeCov

.github/workflows/publish-release-to-dockerhub-workflow.yaml

@@ -0,0 +1,58 @@
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+name: Publish Docker images
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build_and_test:
+    uses: ./.github/workflows/build-workflow.yaml
+    secrets: inherit
+
+  push_release_to_registry:
+    name: Push Docker images to Docker Hub
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: true
+    needs:
+    - build_and_test
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: |
+            astarte/vernemq
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Build and push tagged Docker image
+        id: push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.github/workflows/publish-snapshot-to-dockerhub-workflow.yaml

@@ -0,0 +1,70 @@
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+name: Publish snapshot Docker images
+
+on:
+  push:
+    paths:
+    - '.github/workflows/publish-snapshot-to-dockerhub-workflow.yaml'
+    - 'Dockerfile'
+    branches:
+    - 'master'
+    - 'release-*'
+
+jobs:
+  build_and_test:
+    uses: ./.github/workflows/build-workflow.yaml
+    secrets: inherit
+
+  push_snapshot_to_registry:
+    name: Push Docker images to Docker Hub
+    runs-on: ubuntu-22.04
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
+    strategy:
+      fail-fast: true
+    needs:
+    - build_and_test
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Compute tag name for snapshot images
+        id: compute-tag
+        run: |
+          export TAG="$(echo ${{ github.ref }} | sed 's,refs/heads/,,' | sed 's/master/snapshot/g' | sed 's/release-\(.*\)/\1-snapshot/g' )"
+          echo "TAG=$TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: |
+            astarte/vernemq
+          tags: |
+            # TODO we probably want something smarter, but the 'pattern' type runs only on tags at the moment
+            type=raw,value=${{ steps.compute-tag.outputs.TAG }}
+
+      - name: Build and push tagged Docker image
+        id: push
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64

.tool-versions

@@ -1,2 +1,2 @@
-elixir 1.14.5-otp-25
-erlang 25.3.2
+elixir 1.15.7-otp-26
+erlang 26.1

CHANGELOG.md

@@ -4,6 +4,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+### Added
+- The plugin now accesses the Astarte database. The following
+  env variables have been added:
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__NODES`
+  (defaults to `localhost:9042`)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__USERNAME`
+  (defaults to `cassandra`)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__PASSWORD`
+  (defaults to `cassandra`)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__POOL_SIZE`
+  (defaults to 10)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__SSL_ENABLED`
+  (defaults to `false`)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__SSL_DISABLE_SNI`
+  (defaults to `true`)
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__SSL_CUSTOM_SNI`
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__SSL_CA_FILE`
+- Added support for device deletion. During deletion, a device is
+  disconnected and not allowed to reconnect until deletion ends.
+  Inflight messages are discarded. After deletion, a device must be
+  registered again in order to connect to Astarte.
+- Added support for multiple Astarte instances sharing the same database,
+  the following env variable has been added:
+  - `DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__ASTARTE_INSTANCE_ID`
+  (defaults to ``)
+- Added support for `capabilities` message topic at `/<realm name>/<device name>/capabilities`
+
+### Changed
+- Update Elixir to 1.15.7.
+- Update Erlang/OTP to 26.1.
+- Update VerneMQ to master (1cc57fa) to support OTP 26.
+
 ## [1.1.1] - 2023-10-03
 
 ## [1.1.0] - 2023-06-20
@@ -18,6 +51,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Correctly serialize disconnection/reconnection events if VerneMQ hooks are called in
   the wrong order. Fix https://github.com/astarte-platform/astarte/issues/668.
 
+## [1.0.6] - 2024-04-18
+
 ## [1.0.5] - 2023-09-25
 
 ## [1.0.4] - 2022-09-26

Dockerfile

@@ -1,4 +1,4 @@
-FROM hexpm/elixir:1.14.5-erlang-25.3.2-debian-bullseye-20230522-slim as builder
+FROM --platform=${BUILDPLATFORM} hexpm/elixir:1.15.5-erlang-26.1-debian-bullseye-20230612-slim AS builder
 
 # install build dependencies
 # --allow-releaseinfo-change allows to pull from 'oldstable'
@@ -8,13 +8,15 @@ RUN apt-get update --allow-releaseinfo-change -y \
 
 WORKDIR /build
 
-# Needed for VerneMQ 1.13.0
+# TODO check if it's still needed for VerneMQ 2.0.1
 RUN apt-get -qq update && apt-get -qq install libsnappy-dev libssl-dev
 
 # Let's start by building VerneMQ
-RUN git clone https://github.com/vernemq/vernemq.git -b 1.13.0
+RUN git clone https://github.com/vernemq/vernemq.git
 
 RUN cd vernemq && \
+  # Check out latest master
+  git checkout -b v2.0.1 && \
   make rel && \
   cd ..
 
@@ -48,53 +50,57 @@ RUN cp astarte_vmq_plugin/priv/astarte_vmq_plugin.schema vernemq/_build/default/
 
 # Copy configuration files here - mainly because we want to keep the target image as small as possible
 # and avoid useless layers.
-COPY docker/files/vm.args /build/vernemq/_build/default/rel/vernemq/etc/
-COPY docker/files/vernemq.conf /build/vernemq/_build/default/rel/vernemq/etc/
-COPY docker/bin/rand_cluster_node.escript /build/vernemq/_build/default/rel/vernemq/bin/
-COPY docker/bin/vernemq.sh /build/vernemq/_build/default/rel/vernemq/bin/
-RUN chmod +x /build/vernemq/_build/default/rel/vernemq/bin/vernemq.sh
+COPY docker/files/vm.args vernemq/_build/default/rel/vernemq/etc/
+COPY docker/files/vernemq.conf vernemq/_build/default/rel/vernemq/etc/
+COPY docker/bin/rand_cluster_node.escript vernemq/_build/default/rel/vernemq/bin/
 
 # Note: it is important to keep Debian versions in sync, or incompatibilities between libcrypto will happen
-FROM debian:bullseye-slim
+FROM --platform=${BUILDPLATFORM} debian:bullseye-slim
 
 # Set the locale
 ENV LANG C.UTF-8
 
 # We have to redefine this here since it goes out of scope for each build stage
 ARG BUILD_ENV=prod
 
+# Install some VerneMQ scripts dependencies
+RUN apt-get -qq update && apt-get -qq install bash procps openssl iproute2 curl jq libsnappy-dev net-tools nano
+
 # We need SSL, curl, iproute2 and jq - and to ensure /etc/ssl/astarte
-RUN apt-get -qq update && apt-get -qq install libssl1.1 curl jq iproute2 netcat libsnappy1v5 && apt-get clean && mkdir -p /etc/ssl/astarte
+# TODO some of these might not be needed anymore
+RUN apt-get -qq update && apt-get -qq install libssl1.1 curl jq iproute2 netcat && apt-get clean && mkdir -p /etc/ssl/astarte
 
-# Copy our built stuff (both are self-contained with their ERTS release)
-COPY --from=builder /build/vernemq/_build/default/rel/vernemq /opt/vernemq/
-COPY --from=builder /build/astarte_vmq_plugin/_build/$BUILD_ENV/rel/astarte_vmq_plugin /opt/astarte_vmq_plugin/
+ENV PATH="/opt/vernemq/bin:$PATH"
 
-# Add the wait-for utility
-RUN cd /usr/bin && curl -O https://raw.githubusercontent.com/eficode/wait-for/master/wait-for && chmod +x wait-for && cd -
+COPY --from=builder /build/astarte_vmq_plugin/docker/bin/vernemq.sh /usr/sbin/start_vernemq
+COPY --from=builder /build/astarte_vmq_plugin/docker/bin/join_cluster.sh /usr/sbin/join_cluster
 
-# MQTT
-EXPOSE 1883
+RUN chmod +x /usr/sbin/start_vernemq
+RUN chmod +x /usr/sbin/join_cluster
 
-# MQTT for Reverse Proxy
-EXPOSE 1885
+# Copy our built stuff (both are self-contained with their ERTS release)
+COPY --from=builder /build/vernemq/_build/default/rel/vernemq /opt/vernemq/
 
-# MQTT/SSL
-EXPOSE 8883
+RUN ln -s /opt/vernemq/etc /etc/vernemq && \
+    ln -s /opt/vernemq/data /var/lib/vernemq && \
+    ln -s /opt/vernemq/log /var/log/vernemq
 
-# VerneMQ Message Distribution
-EXPOSE 44053
+COPY --from=builder /build/astarte_vmq_plugin/_build/$BUILD_ENV/rel/astarte_vmq_plugin /opt/astarte_vmq_plugin/
 
-# EPMD - Erlang Port Mapper Daemon
-EXPOSE 4369
+# Ports
+# 1883  MQTT
+# 8883  MQTT/SSL
+# 8080  MQTT WebSockets
+# 44053 VerneMQ Message Distribution
+# 4369  EPMD - Erlang Port Mapper Daemon
+# 8888  Health, API, Prometheus Metrics
+# 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109  Specific Distributed Erlang Port Range
 
-# Specific Distributed Erlang Port Range
-EXPOSE 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109
+EXPOSE 1883 8883 8080 44053 4369 8888 \
+       9100 9101 9102 9103 9104 9105 9106 9107 9108 9109
 
-# Prometheus Metrics
-EXPOSE 8888
+VOLUME ["/opt/vernemq/log", "/opt/vernemq/data", "/opt/vernemq/etc"]
 
-# Expose port for webroot ACME verification (in case)
-EXPOSE 80
+HEALTHCHECK CMD vernemq ping | grep -q pong
 
-CMD ["/opt/vernemq/bin/vernemq.sh"]
+CMD ["start_vernemq"]

config/config.exs

@@ -20,7 +20,4 @@
 # and its dependencies with the aid of the Mix.Config module.
 import Config
 
-config :lager,
-  handlers: [level: :critical]
-
 import_config "#{config_env()}.exs"

config/test.exs

@@ -1,7 +1,7 @@
 #
 # This file is part of Astarte.
 #
-# Copyright 2017 Ispirata Srl
+# Copyright 2017 - 2023 SECO Mind Srl
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,17 +18,8 @@
 
 import Config
 
-# lager is used by rabbit_common.
-# Silent it by setting the higher loglevel.
-config :lager,
-  error_logger_redirect: false,
-  handlers: [level: :critical]
-
 config :astarte_vmq_plugin, :amqp_options, host: System.get_env("RABBITMQ_HOST") || "rabbitmq"
 
-# make amqp supervisors logs less verbose
-config :logger, handle_otp_reports: false
-
 config :logger, :console,
   format: {PrettyLog.UserFriendlyFormatter, :format},
   metadata: [:function]