OCPBUGS-44695: Add signature copy and verification support when working with local Quay registry

test/assets/kustomizations/base/pod-base.yaml

@@ -9,7 +9,7 @@ spec:
       type: RuntimeDefault
   containers:
   - name: test-container
-    image: nginxinc/nginx-unprivileged:latest
+    image: docker.io/nginxinc/nginx-unprivileged:latest
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/assets/reboot/pod-with-pvc.yaml

@@ -21,7 +21,7 @@ spec:
       type: RuntimeDefault
   containers:
   - name: test-container
-    image: nginxinc/nginx-unprivileged:latest
+    image: docker.io/nginxinc/nginx-unprivileged:latest
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:

test/bin/mirror_registry.sh

@@ -18,10 +18,30 @@ setup_prereqs() {
         "${SCRIPTDIR}/../../scripts/dnf_retry.sh" "install" "podman skopeo jq"
     fi
 
+    # Create registry repository base directory structure
+    mkdir -p "${MIRROR_REGISTRY_DIR}"
+    mkdir -p "${QUAY_CONFIG_DIR}"
+
+    # Create a new pull secret file containing authentication information for both
+    # remote (from PULL_SECRET environment) and local registries
+    cat > "${QUAY_CONFIG_DIR}/microshift_auth.json" <<EOF
+{
+    "auths": {
+        "${MIRROR_REGISTRY_URL}": {
+            "auth": "$(echo -n 'microshift:microshift' | base64)"
+        }
+    }
+}
+EOF
+    jq -s '.[0] * .[1]' "${PULL_SECRET}" "${QUAY_CONFIG_DIR}/microshift_auth.json" > "${QUAY_CONFIG_DIR}/pull_secret.json"
+    chmod 600 "${QUAY_CONFIG_DIR}/pull_secret.json"
+    # Reset the pull secret variable to point to the new file
+    PULL_SECRET="${QUAY_CONFIG_DIR}/pull_secret.json"
+
     # TLS authentication is disabled in Quay local registry. The mirror-images.sh
     # helper uses skopeo without TLS options and it defaults to https, so we need
     # to configure registries.conf.d for skopeo to try http instead.
-    sudo bash -c 'cat > /etc/containers/registries.conf.d/900-microshift-mirror.conf' << EOF
+    sudo bash -c 'cat > /etc/containers/registries.conf.d/900-microshift-mirror.conf' <<EOF
 [[registry]]
     prefix = ""
     location = "${MIRROR_REGISTRY_URL}"
@@ -42,25 +62,41 @@ setup_prereqs() {
     insecure = true
 EOF
 
-    # Create registry repository base directory structure
-    mkdir -p "${MIRROR_REGISTRY_DIR}"
-    mkdir -p "${QUAY_CONFIG_DIR}"
+# Complete the source registry configuration to use sigstore attachments.
+# Note that registry.redhat.io.yaml file already exists, but it is missing the
+# sigstore attachment enablement setting.
+sudo bash -c 'cat > /etc/containers/registries.d/registry.quay.io.yaml' <<'EOF'
+docker:
+    quay.io:
+        use-sigstore-attachments: true
+EOF
 
-    # Create a new pull secret file containing authentication information for both
-    # remote (from PULL_SECRET environment) and local registries
-    cat > "${QUAY_CONFIG_DIR}/microshift_auth.json" <<EOF
-{
-    "auths": {
-        "${MIRROR_REGISTRY_URL}": {
-            "auth": "$(echo -n 'microshift:microshift' | base64)"
-        }
-    }
-}
+if [   -e /etc/containers/registries.d/registry.redhat.io.yaml ] &&
+   [ ! -e /etc/containers/registries.d/registry.redhat.io.yaml.orig ]; then
+   sudo mv /etc/containers/registries.d/registry.redhat.io.yaml /etc/containers/registries.d/registry.redhat.io.yaml.orig
+fi
+
+sudo bash -c 'cat > /etc/containers/registries.d/registry.redhat.io.yaml' <<'EOF'
+docker:
+    registry.redhat.io:
+        use-sigstore-attachments: true
+        sigstore: https://registry.redhat.io/containers/sigstore
+EOF
+
+# Configure the destination local registry to use sigstore attachments.
+# Note: The sigstore staging directory is required because not all registries
+# support direct copy of signatures. In this case, the signatures are downloaded
+# locally and copied to the destination registry.
+local -r quay_base="$(dirname "${MIRROR_REGISTRY_URL}")"
+local -r sigstore="${MIRROR_REGISTRY_DIR}/sigstore-staging"
+
+mkdir -p "${sigstore}"
+sudo bash -c 'cat > /etc/containers/registries.d/registry.quay.local.yaml' <<EOF
+docker:
+    ${quay_base}:
+        use-sigstore-attachments: true
+        lookaside-staging: file://${sigstore}
 EOF
-    jq -s '.[0] * .[1]' "${PULL_SECRET}" "${QUAY_CONFIG_DIR}/microshift_auth.json" > "${QUAY_CONFIG_DIR}/pull_secret.json"
-    chmod 600 "${QUAY_CONFIG_DIR}/pull_secret.json"
-    # Reset the pull secret variable to point to the new file
-    PULL_SECRET="${QUAY_CONFIG_DIR}/pull_secret.json"
 }
 
 setup_registry() {

test/bin/pyutils/build_bootc_images.py

@@ -393,6 +393,7 @@ def ostree_rev_in_registry(ce_imgref):
         try:
             dst_ref_cmd = [
                 "skopeo", "inspect",
+                "--authfile", PULL_SECRET,
                 f"docker://{ce_targetimg}",
                 "2>/dev/null", "|",
                 "jq", "-r", "'.Labels[\"ostree.commit\"]'"
@@ -422,9 +423,12 @@ def ostree_rev_in_registry(ce_imgref):
                 common.record_junit(ce_path, "process-container-encapsulate", "SKIPPED")
                 return
 
-            # Run the container image build command
+            # Run the container image build command.
+            # The REGISTRY_AUTH_FILE setting is required for skopeo to succeed
+            # in accessing container registries that might require authentication.
             build_args = [
-                "sudo", "rpm-ostree", "compose",
+                "sudo", f"REGISTRY_AUTH_FILE={PULL_SECRET}",
+                "rpm-ostree", "compose",
                 "container-encapsulate",
                 "--repo", os.path.join(IMAGEDIR, "repo"),
                 ce_imgref,
@@ -438,6 +442,7 @@ def ostree_rev_in_registry(ce_imgref):
             # necessary for subsequent builds that depend on this container image
             copy_args = [
                 "sudo", "skopeo", "copy",
+                "--authfile", PULL_SECRET,
                 f"docker://{ce_targetimg}",
                 f"containers-storage:{ce_localimg}"
             ]

test/bin/scenario.sh

@@ -26,6 +26,9 @@ VM_BOOT_TIMEOUT=1200 # Overall total boot times are around 15m
 VM_GREENBOOT_TIMEOUT=1800 # Greenboot readiness may take up to 15-30m depending on the load
 SKIP_SOS=${SKIP_SOS:-false}  # may be overridden in global settings file
 SKIP_GREENBOOT=${SKIP_GREENBOOT:-false}  # may be overridden in scenario file
+# Container image signature verification should be disabled by default in the
+# main branch because not all the images are signed
+IMAGE_SIGSTORE_ENABLED=false # may be overridden in scenario file
 VNC_CONSOLE=${VNC_CONSOLE:-false}  # may be overridden in global settings file
 TEST_RANDOMIZATION="all"  # may be overridden in scenario file
 TEST_EXECUTION_TIMEOUT="30m" # may be overriden in scenario file
@@ -300,6 +303,7 @@ prepare_kickstart() {
             -e "s|REPLACE_MIRROR_HOSTNAME|${hostname}|g" \
             -e "s|REPLACE_MIRROR_PORT|${MIRROR_REGISTRY_PORT}|g" \
             -e "s|REPLACE_VM_BRIDGE_IP|${VM_BRIDGE_IP}|g" \
+            -e "s|REPLACE_IMAGE_SIGSTORE_ENABLED|${IMAGE_SIGSTORE_ENABLED}|g" \
             "${ifile}" > "${output_file}"
     done
     record_junit "${vmname}" "prepare_kickstart" "OK"

test/kickstart-templates/includes/post-containers-sigstore.cfg

@@ -0,0 +1,79 @@
+# Red Hat public keys are published at https://access.redhat.com/security/team/key.
+# Currently release key 3 is used.
+cat > /etc/containers/RedHat_ReleaseKey3.pub <<'EOF'
+pub   4096R/E60D446E63405576 2024-09-20
+uid                          Red Hat, Inc. (release key 3) <[email protected]>
+
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0ASyuH2TLWvBUqPHZ4Ip
+75g7EncBkgQHdJnjzxAW5KQTMh/siBoB/BoSrtiPMwnChbTCnQOIQeZuDiFnhuJ7
+M/D3b7JoX0m123NcCSn67mAdjBa6Bg6kukZgCP4ZUZeESajWX/EjylFcRFOXW57p
+RDCEN42J/jYlVqt+g9+Grker8Sz86H3l0tbqOdjbz/VxHYhwF0ctUMHsyVRDq2QP
+tqzNXlmlMhS/PoFr6R4u/7HCn/K+LegcO2fAFOb40KvKSKKVD6lewUZErhop1CgJ
+XjDtGmmO9dGMF71mf6HEfaKSdy+EE6iSF2A2Vv9QhBawMiq2kOzEiLg4nAdJT8wg
+ZrMAmPCqGIsXNGZ4/Q+YTwwlce3glqb5L9tfNozEdSR9N85DESfQLQEdY3CalwKM
+BT1OEhEX1wHRCU4drMOej6BNW0VtscGtHmCrs74jPezhwNT8ypkyS+T0zT4Tsy6f
+VXkJ8YSHyenSzMB2Op2bvsE3grY+s74WhG9UIA6DBxcTie15NSzKwfzaoNWODcLF
+p7BY8aaHE2MqFxYFX+IbjpkQRfaeQQsouDFdCkXEFVfPpbD2dk6FleaMTPuyxtIT
+gjVEtGQK2qGCFGiQHFd4hfV+eCA63Jro1z0zoBM5BbIIQ3+eVFwt3AlZp5UVwr6d
+secqki/yrmv3Y0dqZ9VOn3UCAwEAAQ==
+-----END PUBLIC KEY-----
+EOF
+
+# Configure Red Hat containers policy to use the Red Hat public key, making
+# an exception for unofficial / unreleased images that are used by tests
+if [ -e /etc/containers/policy.json ] && [ ! -e /etc/containers/policy.json.orig ]; then
+    mv /etc/containers/policy.json /etc/containers/policy.json.orig
+fi
+cat > /etc/containers/policy.json <<'EOF'
+{
+    "default": [
+        {
+            "type": "reject"
+        }
+    ],
+    "transports": {
+        "docker": {
+            "quay.io/openshift-release-dev": [{
+                "type": "sigstoreSigned",
+                "keyPath": "/etc/containers/RedHat_ReleaseKey3.pub",
+                "signedIdentity": {
+                    "type": "matchRepoDigestOrExact"
+                }
+            }],
+            "registry.redhat.io": [{
+                "type": "sigstoreSigned",
+                "keyPath": "/etc/containers/RedHat_ReleaseKey3.pub",
+                "signedIdentity": {
+                    "type": "matchRepoDigestOrExact"
+                }
+            }],
+            "quay.io/microshift": [{
+                "type": "insecureAcceptAnything"
+            }],
+            "quay.io/container-perf-tools": [{
+                "type": "insecureAcceptAnything"
+            }],
+            "registry.ci.openshift.org": [{
+                "type": "insecureAcceptAnything"
+            }],
+            "docker.io": [{
+                "type": "insecureAcceptAnything"
+            }]
+        }
+    }
+}
+EOF
+
+# Configure the MicroShift remote and local registries to use sigstore attachments
+cat > /etc/containers/registries.d/registry.quay.io.yaml <<'EOF'
+docker:
+    quay.io/openshift-release-dev:
+        use-sigstore-attachments: true
+EOF
+
+cat > /etc/containers/registries.d/registry.quay.local.yaml <<'EOF'
+docker:
+    REPLACE_MIRROR_HOSTNAME:REPLACE_MIRROR_PORT:
+        use-sigstore-attachments: true
+EOF

test/kickstart-templates/includes/post-containers.cfg

@@ -1,27 +1,34 @@
 # The pull secret is mandatory for MicroShift builds on top of OpenShift, but not OKD
 # The /etc/crio/crio.conf.d/microshift.conf references the /etc/crio/openshift-pull-secret file
 mkdir -p /etc/crio
-cat > /etc/crio/openshift-pull-secret <<EOF
+cat > /etc/crio/openshift-pull-secret <<'EOF'
 REPLACE_PULL_SECRET
 EOF
 chmod 600 /etc/crio/openshift-pull-secret
 
+# Add the mirror registry host name resolution
+cat >> /etc/hosts <<'EOF'
+REPLACE_VM_BRIDGE_IP REPLACE_MIRROR_HOSTNAME
+EOF
+
 # Setup mirror registries configuration here, as the hostname is dynamic and the file is verbose.
 # Use hostnames as IP addresses are not allowed.
 mkdir -p /etc/containers/registries.conf.d
-cat > /etc/containers/registries.conf.d/999-microshift-mirror.conf <<EOF
+cat > /etc/containers/registries.conf.d/999-microshift-mirror.conf <<'EOF'
 [[registry]]
     prefix = ""
     location = "REPLACE_MIRROR_HOSTNAME:REPLACE_MIRROR_PORT"
     mirror-by-digest-only = true
     insecure = true
+
 [[registry]]
     prefix = ""
     location = "quay.io"
     mirror-by-digest-only = true
 [[registry.mirror]]
     location = "REPLACE_MIRROR_HOSTNAME:REPLACE_MIRROR_PORT/microshift"
     insecure = true
+
 [[registry]]
     prefix = ""
     location = "registry.redhat.io"
@@ -31,9 +38,13 @@ cat > /etc/containers/registries.conf.d/999-microshift-mirror.conf <<EOF
     insecure = true
 EOF
 
-# Skip signature verifying for red hat registries, since the signatures are bound the original
-# registry name and mirroring images changes that.
-cat > /etc/containers/policy.json <<EOF
+# Skip signature verification for all images by default.
+# Tests that support signature verification will overwrite this file and reenable
+# it for selected Red Hat registries.
+if [ -e /etc/containers/policy.json ] && [ ! -e /etc/containers/policy.json.orig ]; then
+    mv /etc/containers/policy.json /etc/containers/policy.json.orig
+fi
+cat > /etc/containers/policy.json <<'EOF'
 {
     "default": [
         {
@@ -49,7 +60,3 @@ cat > /etc/containers/policy.json <<EOF
         }
 }
 EOF
-
-cat >> /etc/hosts <<EOF
-REPLACE_VM_BRIDGE_IP REPLACE_MIRROR_HOSTNAME
-EOF

test/kickstart-templates/kickstart-bootc.ks.template

@@ -6,7 +6,12 @@
 %post --log=/dev/console --erroronfail
 
 %include /post-microshift.cfg
+
 %include /post-containers.cfg
+if REPLACE_IMAGE_SIGSTORE_ENABLED ; then
+%include /post-containers-sigstore.cfg
+fi
+
 %include /post-system.cfg
 %include /post-network.cfg
 

test/kickstart-templates/kickstart-centos.ks.template

@@ -5,7 +5,12 @@
 %post --log=/dev/console --erroronfail
 
 %include /post-microshift.cfg
+
 %include /post-containers.cfg
+if REPLACE_IMAGE_SIGSTORE_ENABLED ; then
+%include /post-containers-sigstore.cfg
+fi
+
 %include /post-system.cfg
 %include /post-network.cfg
 

test/kickstart-templates/kickstart-liveimg.ks.template

@@ -5,7 +5,12 @@
 %post --log=/dev/console --erroronfail
 
 %include /post-microshift.cfg
+
 %include /post-containers.cfg
+if REPLACE_IMAGE_SIGSTORE_ENABLED ; then
+%include /post-containers-sigstore.cfg
+fi
+
 %include /post-system.cfg
 %include /post-network.cfg
 %include /post-fips.cfg

test/kickstart-templates/kickstart.ks.template

@@ -5,7 +5,12 @@
 %post --log=/dev/console --erroronfail
 
 %include /post-microshift.cfg
+
 %include /post-containers.cfg
+if REPLACE_IMAGE_SIGSTORE_ENABLED ; then
+%include /post-containers-sigstore.cfg
+fi
+
 %include /post-system.cfg
 %include /post-network.cfg
 %include /post-fips.cfg

test/scenarios-bootc/periodics/cos9-src@optional.sh

@@ -6,6 +6,9 @@
 VM_BRIDGE_IP="$(get_vm_bridge_ip "${VM_MULTUS_NETWORK}")"
 # shellcheck disable=SC2034  # used elsewhere
 WEB_SERVER_URL="http://${VM_BRIDGE_IP}:${WEB_SERVER_PORT}"
+# Disable signature verification due to unsigned images used in this test
+# shellcheck disable=SC2034  # used elsewhere
+IMAGE_SIGSTORE_ENABLED=false
 
 scenario_create_vms() {
     prepare_kickstart host1 kickstart-bootc.ks.template cos9-bootc-source-optionals

test/scenarios-bootc/periodics/el94-prel@el95-crel@upgrade-ok.sh

@@ -2,6 +2,11 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
+# Disable signature verification because the test performs an upgrade to
+# a target reference unsigned image that was generated by local builds
+# shellcheck disable=SC2034  # used elsewhere
+IMAGE_SIGSTORE_ENABLED=false
+
 dest_image=rhel95-bootc-crel
 
 scenario_create_vms() {

test/scenarios-bootc/periodics/el94-prel@el95-src@upgrade-fails-and-rolls-back.sh

@@ -2,6 +2,11 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
+# Disable signature verification because the test performs an upgrade to
+# a target reference unsigned image that was generated by local builds
+# shellcheck disable=SC2034  # used elsewhere
+IMAGE_SIGSTORE_ENABLED=false
+
 scenario_create_vms() {
     prepare_kickstart host1 kickstart-bootc.ks.template rhel94-bootc-prel
     launch_vm --boot_blueprint rhel94-bootc

test/scenarios-bootc/periodics/el94-prel@el95-src@upgrade-ostree2bootc-ok.sh

@@ -2,6 +2,11 @@
 
 # Sourced from scenario.sh and uses functions defined there.
 
+# Disable signature verification because the test performs an upgrade to
+# a target reference unsigned image that was generated by local builds
+# shellcheck disable=SC2034  # used elsewhere
+IMAGE_SIGSTORE_ENABLED=false
+
 scenario_create_vms() {
     # The y-1 ostree image will be fetched from the cache as it is not built
     # as part of the bootc image build procedure