[release-4.18] OCPBUGS-48577: Enable sigstore-based verification in MicroShift CI

docs/contributor/howto_quay_build.md

@@ -0,0 +1,87 @@
+# Building Quay From Sources for MicroShift CI
+
+## Introduction
+
+MicroShift CI requires a local mirror registry for storing container images and
+container build artifacts. This is necessary for improving the stability of tests
+(less dependency on network flakes) and overall performance (faster image access).
+
+The simplest local mirror registry tool that can be used in MicroShift CI is
+[Docker Distribution Registry](https://docs.docker.com/registry), but it lacks
+support of `sigstore`, which forces its users not to use signature validation.
+
+[Project Quay](https://github.com/quay/quay) provides for a more feature rich
+alternative, but it comes with a few challenges:
+* Quay registry is more complex to set up and more resource consuming
+* Quay registry is not supported on the `aarch64` platform
+
+Quay can be deployed in a [Quick Local Development](https://github.com/quay/quay/blob/master/docs/quick-local-deployment.md)
+mode that runs the registry inside local containers. To mitigate the lack of the
+`aarch64` platform support, it is necessary to pre-build the required container
+images from [Project Quay](https://github.com/quay/quay) sources and store them
+at [quay.io/microshift] registry to be consumed by MicroShift CI.
+
+The remainder of this document describes how to build Quay container images from
+sources and store them in a cloud registry.
+
+## Prerequisites
+
+Run the following command to initialize the Quay Git repository.
+
+```
+git clone https://github.com/quay/quay.git
+cd quay
+```
+
+Review the list of [Quay Releases](https://github.com/quay/quay/releases) to
+select the branch. It is recommended to use a release with the `latest` tag
+(`v3.11.7` at the time of writing this document).
+
+Check out the appropriate code branch.
+
+```
+QUAY_VER=v3.11.7
+git checkout "${QUAY_VER}"
+```
+
+Install the RPM dependencies required to build Quay images from sources.
+
+```
+sudo dnf install -y podman podman-compose
+sudo ln -s $(which podman-compose) /usr/bin/docker-compose
+```
+
+## Image Build
+
+Run the following command to build Quay container images.
+
+```
+make local-dev-build-images
+```
+
+Make sure that required `quay-local` image was built successfully.
+
+```
+$ podman images quay-local
+REPOSITORY            TAG         IMAGE ID      CREATED        SIZE
+localhost/quay-local  latest      7f4def76a288  2 minutes ago  786 MB
+```
+
+## Image Push
+
+Log into your `quay.io` account at the `microshift` organization.
+
+```
+podman login quay.io/microshift
+```
+
+Tag the local image with the version and current architecture, and push it to
+the cloud registry.
+
+```
+podman tag localhost/quay-local:latest quay.io/microshift/quay:${QUAY_VER}-$(uname -m)
+podman push quay.io/microshift/quay:${QUAY_VER}-$(uname -m)
+```
+
+Finally, browse to [Quay Repository Settings](https://quay.io/repository/microshift/quay?tab=settings)
+and make sure the repository has public access.

scripts/devenv-builder/cleanup-composer.sh

@@ -16,10 +16,10 @@ clean_podman_images() {
 
     title "Cleaning up running containers"
     for id in $(sudo podman ps -a | awk '{print $1}') ; do
-        sudo podman rm -f "${id}"
+        sudo podman rm -f --volumes --time 0 "${id}" || true
     done
     for id in $(podman ps -a | awk '{print $1}') ; do
-        podman rm -f "${id}"
+        podman rm -f --volumes --time 0 "${id}" || true
     done
 
     if [ "${FULL_CLEAN}" = 1 ] ; then

scripts/devenv-builder/configure-composer.sh

@@ -21,7 +21,7 @@ fi
      git composer-cli ostree rpm-ostree \
      cockpit-composer bash-completion podman runc genisoimage \
      createrepo yum-utils selinux-policy-devel jq wget lorax rpm-build \
-     containernetworking-plugins expect"
+     containernetworking-plugins expect httpd-tools"
 
 if grep -qE "Red Hat Enterprise Linux.*Beta" /etc/redhat-release; then
    VID=$(source /etc/os-release && echo "${VERSION_ID}")

test/assets/isolated-netfilter.xml

@@ -0,0 +1,16 @@
+<filter name='${VM_ISOLATED_NETWORK}' chain='root'>
+    <!-- Allow outgoing traffic to private IP ranges and DHCP -->
+    <rule action='accept' direction='out' priority='500'>
+        <ip dstipaddr='192.168.0.0' dstipmask='16'/>
+    </rule>
+    <rule action='accept' direction='out' priority='500'>
+        <ip dstipaddr='10.0.0.0' dstipmask='8'/>
+    </rule>
+    <rule action='accept' direction='inout' priority='500'>
+        <ip srcportstart='67' srcportend='68' protocol='udp'/>
+    </rule>
+    <!-- Drop all other outgoing IP traffic -->
+    <rule action='drop' direction='out' priority='501'>
+        <ip/>
+    </rule>
+</filter>

test/assets/isolated-network.xml

@@ -1,6 +1,11 @@
 <network>
   <name>${VM_ISOLATED_NETWORK}</name>
-  <forward mode='none'/>
+  <!-- An isolated network would ideally have no forwarding. We are using
+       NAT here because Quay registry for mirroring requires traffic forwarding
+       between private IPs in the hypervisor. In order to create the isolation a
+       nwfilter is used to mimick the forward mode none that should be here. Only
+       private IPs and DHCP are allowed. See isolated-netfilter.xml -->
+  <forward mode='nat'/>
   <ip address='192.168.111.1' netmask='255.255.255.0' localPtr='yes'>
     <dhcp>
       <range start='192.168.111.100' end='192.168.111.254'/>

test/assets/kustomizations/base/pod-base.yaml

@@ -9,7 +9,7 @@ spec:
       type: RuntimeDefault
   containers:
   - name: test-container
-    image: nginxinc/nginx-unprivileged:latest
+    image: docker.io/nginxinc/nginx-unprivileged:latest
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/assets/quay/config.yaml.template

@@ -0,0 +1,22 @@
+BUILDLOGS_REDIS:
+    host: ${REDIS_IP}
+    password: strongpassword
+    port: 6379
+CREATE_NAMESPACE_ON_PUSH: true
+DATABASE_SECRET_KEY: a8c2744b-7004-4af2-bcee-e417e7bdd235
+DB_URI: postgresql://quayuser:quaypass@${POSTGRES_IP}:5432/quay
+DISTRIBUTED_STORAGE_CONFIG:
+    default:
+        - LocalStorage
+        - storage_path: /datastorage/registry
+DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: []
+DISTRIBUTED_STORAGE_PREFERENCE:
+    - default
+FEATURE_MAILING: false
+SECRET_KEY: e9bd34f4-900c-436a-979e-7530e5d74ac8
+SERVER_HOSTNAME: ${QUAY_URL}
+SETUP_COMPLETE: true
+USER_EVENTS_REDIS:
+    host: ${REDIS_IP}
+    password: strongpassword
+    port: 6379

test/assets/reboot/pod-with-pvc.yaml

@@ -21,7 +21,7 @@ spec:
       type: RuntimeDefault
   containers:
   - name: test-container
-    image: nginxinc/nginx-unprivileged:latest
+    image: docker.io/nginxinc/nginx-unprivileged:latest
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/bin/common.sh

@@ -191,7 +191,7 @@ WEB_SERVER_URL="http://${VM_BRIDGE_IP:-$(hostname)}:${WEB_SERVER_PORT}"
 export MIRROR_REGISTRY_PORT=5000
 
 # Mirror registry URL using VM bridge IP with fallback to host name
-MIRROR_REGISTRY_URL="${VM_BRIDGE_IP:-$(hostname)}:${MIRROR_REGISTRY_PORT}"
+MIRROR_REGISTRY_URL="${VM_BRIDGE_IP:-$(hostname)}:${MIRROR_REGISTRY_PORT}/microshift"
 export MIRROR_REGISTRY_URL
 
 get_build_branch() {

test/bin/manage_hypervisor_config.sh

@@ -14,8 +14,8 @@ ${BASH_SOURCE[0]} (create|cleanup|cleanup-all)
 
   -h           Show this help.
 
-create: Set up firewall, storage pool and network. 
-        Start nginx file-server to serve images 
+create: Set up firewall, storage pool and network.
+        Start nginx file-server to serve images
         for test scenarios.
         Uses the VM_POOL_BASENAME, VM_DISK_BASEDIR and
         VM_ISOLATED_NETWORK variables.
@@ -30,6 +30,11 @@ EOF
 firewall_settings() {
     local -r action=$1
 
+    # Web server port to allow access from virtual machines
+    sudo firewall-cmd --permanent --zone=public "--${action}-port"="${WEB_SERVER_PORT}/tcp"
+    # VNC ports for remote console connection to virtual machines
+    sudo firewall-cmd --permanent --zone=public "--${action}-port=5900-5999/tcp"
+    # Enable mDNS over libvirt network
     sudo firewall-cmd --permanent --zone=libvirt "--${action}-service=mdns"
 
     for netname in default "${VM_ISOLATED_NETWORK}" "${VM_MULTUS_NETWORK}" "${VM_IPV6_NETWORK}" "${VM_DUAL_STACK_NETWORK}"; do
@@ -43,9 +48,8 @@ firewall_settings() {
         for ip in $(ip addr show "${vm_bridge}" | grep "scope global" | awk '{print $2}'); do
             sudo firewall-cmd --permanent --zone=trusted "--${action}-source"="${ip}"
         done
-        sudo firewall-cmd --permanent --zone=public  "--${action}-port"="${WEB_SERVER_PORT}/tcp"
-        sudo firewall-cmd --reload
     done
+    sudo firewall-cmd --reload
 }
 
 action_create() {
@@ -60,20 +64,32 @@ action_create() {
 
     # Isolated network
     if ! sudo sudo virsh net-info "${VM_ISOLATED_NETWORK}" &>/dev/null ; then
+        # Isolated network creates a NAT network which is later on constrained
+        # for outgoing traffic to only the hypervisor and the virtual machines
+        # This is achieved using a nwfilter, which is created here and specified
+        # in scenario.sh.
         local -r netconfig_tmpl="${SCRIPTDIR}/../assets/isolated-network.xml"
-        local -r netconfig_file="${IMAGEDIR}/infra/isolated-network.xml"
+        local -r netconfig_file="${IMAGEDIR}/isolated-network.xml"
 
         mkdir -p "$(dirname "${netconfig_file}")"
         envsubst <"${netconfig_tmpl}" >"${netconfig_file}"
 
         sudo virsh net-define    "${netconfig_file}"
         sudo virsh net-start     "${VM_ISOLATED_NETWORK}"
         sudo virsh net-autostart "${VM_ISOLATED_NETWORK}"
+
+        local -r netfilter_tmpl="${SCRIPTDIR}/../assets/isolated-netfilter.xml"
+        local -r netfilter_file="${IMAGEDIR}/isolated-netfilter.xml"
+
+        mkdir -p "$(dirname "${netfilter_file}")"
+        envsubst <"${netfilter_tmpl}" >"${netfilter_file}"
+
+        sudo virsh nwfilter-define "${netfilter_file}"
     fi
 
     if ! sudo sudo virsh net-info "${VM_MULTUS_NETWORK}" &>/dev/null ; then
         local -r multus_netconfig_tmpl="${SCRIPTDIR}/../assets/multus-network.xml"
-        local -r multus_netconfig_file="${IMAGEDIR}/infra/multus-network.xml"
+        local -r multus_netconfig_file="${IMAGEDIR}/multus-network.xml"
 
         mkdir -p "$(dirname "${multus_netconfig_file}")"
         envsubst <"${multus_netconfig_tmpl}" >"${multus_netconfig_file}"
@@ -86,7 +102,7 @@ action_create() {
     # IPv6 network
     if ! sudo sudo virsh net-info "${VM_IPV6_NETWORK}" &>/dev/null ; then
         local -r ipv6_netconfig_tmpl="${SCRIPTDIR}/../assets/ipv6-network.xml"
-        local -r ipv6_netconfig_file="${IMAGEDIR}/infra/ipv6-network.xml"
+        local -r ipv6_netconfig_file="${IMAGEDIR}/ipv6-network.xml"
 
         mkdir -p "$(dirname "${ipv6_netconfig_file}")"
         envsubst <"${ipv6_netconfig_tmpl}" >"${ipv6_netconfig_file}"
@@ -103,7 +119,7 @@ action_create() {
 
     if ! sudo sudo virsh net-info "${VM_DUAL_STACK_NETWORK}" &>/dev/null ; then
         local -r dual_stack_netconfig_tmpl="${SCRIPTDIR}/../assets/dual-stack-network.xml"
-        local -r dual_stack_netconfig_file="${IMAGEDIR}/infra/dual-stack-network.xml"
+        local -r dual_stack_netconfig_file="${IMAGEDIR}/dual-stack-network.xml"
 
         mkdir -p "$(dirname "${dual_stack_netconfig_file}")"
         envsubst <"${dual_stack_netconfig_tmpl}" >"${dual_stack_netconfig_file}"
@@ -133,6 +149,7 @@ action_cleanup() {
     if sudo virsh net-info "${VM_ISOLATED_NETWORK}" &>/dev/null ; then
         sudo virsh net-destroy "${VM_ISOLATED_NETWORK}"
         sudo virsh net-undefine "${VM_ISOLATED_NETWORK}"
+        sudo virsh nwfilter-undefine "${VM_ISOLATED_NETWORK}"
     fi
 
     if sudo virsh net-info "${VM_IPV6_NETWORK}" &>/dev/null ; then
@@ -150,7 +167,7 @@ action_cleanup() {
     fi
 
     # Storage pool
-    for pool_name in $(sudo virsh pool-list --name | awk '/vm-storage/ {print $1}') ; do       
+    for pool_name in $(sudo virsh pool-list --name | awk '/vm-storage/ {print $1}') ; do
         if sudo virsh pool-info "${pool_name}" &>/dev/null; then
             sudo virsh pool-destroy "${pool_name}"
             sudo virsh pool-undefine "${pool_name}"

test/bin/manage_webserver.sh

@@ -24,7 +24,7 @@ EOF
 
 action_stop() {
     echo "Stopping web server"
-    sudo pkill nginx || true
+    pkill -U "$(id -u)" nginx || true
     exit 0
 }
 
@@ -70,8 +70,8 @@ EOF
     sudo chgrp -R "$(id -gn)" /var/lib/nginx
 
     # Kill running nginx processes and wait until down
-    sudo pkill nginx || true
-    while pidof nginx &>/dev/null ; do
+    pkill -U "$(id -u)" nginx || true
+    while pgrep -U "$(id -u)" nginx &>/dev/null ; do
         sleep 1
     done