OCPBUGS-44695: Optimize Quay configuration procedure

scripts/devenv-builder/cleanup-composer.sh

@@ -16,10 +16,10 @@ clean_podman_images() {
 
     title "Cleaning up running containers"
     for id in $(sudo podman ps -a | awk '{print $1}') ; do
-        sudo podman rm -f "${id}"
+        sudo podman rm -f --volumes --time 0 "${id}" || true
     done
     for id in $(podman ps -a | awk '{print $1}') ; do
-        podman rm -f "${id}"
+        podman rm -f --volumes --time 0 "${id}" || true
     done
 
     if [ "${FULL_CLEAN}" = 1 ] ; then

test/assets/quay/config.yaml.template

@@ -1,95 +1,22 @@
-ALLOWED_OCI_ARTIFACT_TYPES:
-    application/vnd.oci.image.config.v1+json:
-        - application/vnd.oci.image.layer.v1.tar+zstd
-    application/vnd.sylabs.sif.config.v1+json:
-        - application/vnd.sylabs.sif.layer.v1+tar
-AUTHENTICATION_TYPE: Database
-AVATAR_KIND: local
 BUILDLOGS_REDIS:
     host: ${REDIS_IP}
     password: strongpassword
     port: 6379
-DATABASE_SECRET_KEY: 5861fd40-0eab-42fb-8837-574ae401fa89
-DB_CONNECTION_ARGS: {}
-DB_URI: postgresql://user:pass@${POSTGRES_IP}/quay
-DEFAULT_TAG_EXPIRATION: 2w
+CREATE_NAMESPACE_ON_PUSH: true
+DATABASE_SECRET_KEY: a8c2744b-7004-4af2-bcee-e417e7bdd235
+DB_URI: postgresql://quayuser:quaypass@${POSTGRES_IP}:5432/quay
 DISTRIBUTED_STORAGE_CONFIG:
     default:
         - LocalStorage
         - storage_path: /datastorage/registry
 DISTRIBUTED_STORAGE_DEFAULT_LOCATIONS: []
 DISTRIBUTED_STORAGE_PREFERENCE:
     - default
-FEATURE_ACI_CONVERSION: false
-FEATURE_ACTION_LOG_ROTATION: false
-FEATURE_ANONYMOUS_ACCESS: true
-FEATURE_APP_REGISTRY: false
-FEATURE_APP_SPECIFIC_TOKENS: true
-FEATURE_BITBUCKET_BUILD: false
-FEATURE_BLACKLISTED_EMAILS: false
-FEATURE_BUILD_SUPPORT: false
-FEATURE_CHANGE_TAG_EXPIRATION: true
-FEATURE_DIRECT_LOGIN: true
-FEATURE_EXTENDED_REPOSITORY_NAMES: true
-FEATURE_FIPS: false
-FEATURE_GITHUB_BUILD: false
-FEATURE_GITHUB_LOGIN: false
-FEATURE_GITLAB_BUILD: false
-FEATURE_GOOGLE_LOGIN: false
-FEATURE_INVITE_ONLY_USER_CREATION: false
 FEATURE_MAILING: false
-FEATURE_NONSUPERUSER_TEAM_SYNCING_SETUP: false
-FEATURE_PARTIAL_USER_AUTOCOMPLETE: true
-FEATURE_PROXY_STORAGE: false
-FEATURE_REPO_MIRROR: false
-FEATURE_REQUIRE_TEAM_INVITE: true
-FEATURE_RESTRICTED_V1_PUSH: true
-FEATURE_SECURITY_NOTIFICATIONS: false
-FEATURE_SECURITY_SCANNER: false
-FEATURE_STORAGE_REPLICATION: false
-FEATURE_TEAM_SYNCING: false
-FEATURE_USER_CREATION: true
-FEATURE_USER_LAST_ACCESSED: true
-FEATURE_USER_LOG_ACCESS: false
-FEATURE_USER_METADATA: false
-FEATURE_USER_RENAME: false
-FEATURE_USERNAME_CONFIRMATION: true
-FRESH_LOGIN_TIMEOUT: 10m
-GITHUB_LOGIN_CONFIG: {}
-GITHUB_TRIGGER_CONFIG: {}
-GITLAB_TRIGGER_KIND: {}
-LDAP_ALLOW_INSECURE_FALLBACK: false
-LDAP_EMAIL_ATTR: mail
-LDAP_UID_ATTR: uid
-LDAP_URI: ldap://localhost
-LOG_ARCHIVE_LOCATION: default
-LOGS_MODEL: database
-LOGS_MODEL_CONFIG: {}
-MAIL_DEFAULT_SENDER: [email protected]
-MAIL_PORT: 587
-MAIL_USE_AUTH: false
-MAIL_USE_TLS: false
-PREFERRED_URL_SCHEME: http
-REGISTRY_TITLE: Project Quay
-REGISTRY_TITLE_SHORT: Project Quay
-REPO_MIRROR_INTERVAL: 30
-REPO_MIRROR_TLS_VERIFY: true
-SEARCH_MAX_RESULT_PAGE_COUNT: 10
-SEARCH_RESULTS_PER_PAGE: 10
-SECRET_KEY: f8d15d8b-5b24-4210-a762-1db6175cba2a
-SECURITY_SCANNER_INDEXING_INTERVAL: 30
+SECRET_KEY: e9bd34f4-900c-436a-979e-7530e5d74ac8
 SERVER_HOSTNAME: ${QUAY_URL}
 SETUP_COMPLETE: true
-SUPER_USERS:
-    - microshift
-TAG_EXPIRATION_OPTIONS:
-    - 2w
-TEAM_RESYNC_STALE_TIME: 30m
-TESTING: false
-USE_CDN: false
 USER_EVENTS_REDIS:
     host: ${REDIS_IP}
     password: strongpassword
     port: 6379
-USER_RECOVERY_TOKEN_LIFETIME: 30m
-USERFILES_LOCATION: default

test/bin/mirror_registry.sh

@@ -64,6 +64,7 @@ EOF
 }
 
 setup_registry() {
+    local -r quay_url="$(hostname):${MIRROR_REGISTRY_PORT}"
     local postgres_ip
     local redis_ip
     local new_db=false
@@ -80,10 +81,10 @@ setup_registry() {
     for n in postgres redis quay ; do
         local cn="microshift-${n}"
         echo "Removing '${cn}' container"
-        sudo podman rm -f "${cn}" || true
+        sudo podman rm -f --time 0 "${cn}" || true
     done
 
-    # Pull the registry images locally
+    # Pull the registry images in background locally
     for i in "${POSTGRES_IMAGE}" "${REDIS_IMAGE}" "${QUAY_IMAGE}" ; do
         echo "Pulling '${i}' image locally"
         sudo skopeo copy \
@@ -92,11 +93,13 @@ setup_registry() {
             --retry-times 3 \
             --preserve-digests \
             "docker://${i}" \
-            "containers-storage:${i}"
+            "containers-storage:${i}" &
     done
+    # Wait until the image pull is complete
+    wait
 
     # Set up Postgres
-    # See https://github.com/quay/quay/blob/master/docs/quick-local-deployment.md#set-up-postgres
+    # See https://docs.projectquay.io/deploy_quay.html#poc-configuring-database
     if [ ! -d "${MIRROR_REGISTRY_DIR}/postgres" ] ; then
         mkdir -p "${MIRROR_REGISTRY_DIR}/postgres"
         setfacl -m u:26:-wx "${MIRROR_REGISTRY_DIR}/postgres"
@@ -105,9 +108,10 @@ setup_registry() {
 
     echo "Running Postgres container"
     sudo podman run -d --rm --name microshift-postgres \
-        -e POSTGRES_USER=user \
-        -e POSTGRES_PASSWORD=pass \
+        -e POSTGRES_USER=quayuser \
+        -e POSTGRES_PASSWORD=quaypass \
         -e POSTGRES_DB=quay \
+        -e POSTGRESQL_ADMIN_PASSWORD=adminpass \
         -p 5432:5432 \
         -v "${MIRROR_REGISTRY_DIR}/postgres:/var/lib/postgresql/data:Z" \
         "${POSTGRES_IMAGE}" >/dev/null
@@ -117,7 +121,7 @@ setup_registry() {
     for i in $(seq 60) ; do
         sleep 1
         if sudo podman exec -it microshift-postgres \
-            /bin/bash -c 'echo "CREATE EXTENSION IF NOT EXISTS pg_trgm" | psql -d quay -U user' >/dev/null ; then
+            /bin/bash -c 'echo "CREATE EXTENSION IF NOT EXISTS pg_trgm" | psql -d quay -U quayuser' >/dev/null ; then
             i=0
             break
         fi
@@ -128,7 +132,7 @@ setup_registry() {
     fi
 
     # Setup and run Redis
-    # See https://github.com/quay/quay/blob/master/docs/quick-local-deployment.md#set-up-redis
+    # See https://docs.projectquay.io/deploy_quay.html#poc-configuring-redis
     echo "Running Redis container"
     sudo podman run -d --rm --name microshift-redis \
         -p 6379:6379 \
@@ -137,29 +141,31 @@ setup_registry() {
     redis_ip=$(sudo podman inspect -f "{{.NetworkSettings.IPAddress}}" microshift-redis)
 
     # Set up Quay
-    # See https://docs.projectquay.io/deploy_quay.html#preparing-local-storage
-    if [ ! -d "${MIRROR_REGISTRY_DIR}/storage" ] ; then
-        mkdir -p "${MIRROR_REGISTRY_DIR}/storage"
-        setfacl -m u:1001:-wx "${MIRROR_REGISTRY_DIR}/storage"
-    fi
+    # See https://docs.projectquay.io/deploy_quay.html#poc-deploying-quay
 
-    # Create the configuration from from a template, which was generated according
-    # to the instructions at:
-    # https://github.com/quay/quay/blob/master/docs/quick-local-deployment.md#build-the-quay-configuration-via-configtool
-    # Note: Hardcoded IP and URL must be replaced by respective variables if the
-    # template is regenerated.
+    # Create the configuration template using the minimal configuration settings.
+    # If template is updated, replace hardcoded Postgres, Redis IPs and Quay URL
+    # by respective variables.
+    # See https://docs.projectquay.io/deploy_quay.html#preparing-configuration-file
     POSTGRES_IP="${postgres_ip}" \
     REDIS_IP="${redis_ip}" \
-    QUAY_URL="$(hostname):${MIRROR_REGISTRY_PORT}" \
+    QUAY_URL="${quay_url}" \
     envsubst \
         < "${SCRIPTDIR}/../assets/quay/config.yaml.template" \
         > "${QUAY_CONFIG_DIR}/config.yaml"
 
+    # Enable superuser creation using API
+    # See https://docs.projectquay.io/deploy_quay.html#configuring-superuser
+    cat >> "${QUAY_CONFIG_DIR}/config.yaml" <<EOF
+FEATURE_USER_INITIALIZE: true
+SUPER_USERS:
+    - microshift
+EOF
     # Enable Quay dual-stack server support if the local host supports IPv6
     local podman_network=""
     if ping -6 -c 1 ::1 &>/dev/null ; then
         # Add the configuration option
-        # See https://docs.redhat.com/en/documentation/red_hat_quay/3.11/html-single/configure_red_hat_quay/index?utm_source=chatgpt.com#config-fields-ipv6
+        # See https://docs.redhat.com/en/documentation/red_hat_quay/3/html-single/configure_red_hat_quay/index?utm_source=chatgpt.com#config-fields-ipv6
         echo "FEATURE_LISTEN_IP_VERSION: dual-stack" >> "${QUAY_CONFIG_DIR}/config.yaml"
         # Enable both IPv4 and IPv6 podman container network for the root user
         # See https://access.redhat.com/solutions/6196301
@@ -169,10 +175,16 @@ setup_registry() {
         podman_network="--network=microshift-ipv6-dual-stack"
     fi
 
+    # See https://docs.projectquay.io/deploy_quay.html#preparing-local-storage
+    if [ ! -d "${MIRROR_REGISTRY_DIR}/storage" ] ; then
+        mkdir -p "${MIRROR_REGISTRY_DIR}/storage"
+        setfacl -m u:1001:-wx "${MIRROR_REGISTRY_DIR}/storage"
+    fi
+
     # Run Quay container
-    # See https://github.com/quay/quay/blob/master/docs/quick-local-deployment.md#run-quay
+    # See https://docs.projectquay.io/deploy_quay.html#deploy-quay-registry
     echo "Running Quay container"
-    sudo podman run -d --rm --name=microshift-quay \
+    sudo podman run -d --name=microshift-quay \
         "${podman_network}" \
         -p "${MIRROR_REGISTRY_PORT}:8080" \
         -p "[::]:${MIRROR_REGISTRY_PORT}:8080" \
@@ -183,7 +195,7 @@ setup_registry() {
     # Wait until the Quay instance is started
     for i in $(seq 60) ; do
         sleep 1
-        if curl -sI "${MIRROR_REGISTRY_URL}" &>/dev/null ; then
+        if curl -sI "${quay_url}" 2>/dev/null | grep -Eq "HTTP.*200 OK" ; then
             i=0
             break
         fi
@@ -193,26 +205,26 @@ setup_registry() {
         exit 1
     fi
 
-    # Import the database template content with the 'microshift:microshift' user
-    # definition. The template was exported using the following command:
-    # sudo podman exec -it microshift-postgres /usr/bin/pg_dump --data-only -d quay -U user -t public.user
-    #
-    # Note: Replace the password hash with '$MICROSHIFT_PASSWORD_HASH' string
-    # before committing the template into the source repository.
+    # Create the superuser, verifying the creation was successful
+    # See https://docs.projectquay.io/config_quay.html#using-the-api-to-create-first-user
     if ${new_db} ; then
-        MICROSHIFT_PASSWORD_HASH="$(htpasswd -bnBC 12 "" microshift | tr -d ':\n')" \
-        envsubst \
-            < "${SCRIPTDIR}/../assets/quay/user_dump.sql.template" \
-            > "${QUAY_CONFIG_DIR}/user_dump.sql"
-        sudo podman cp "${QUAY_CONFIG_DIR}/user_dump.sql" microshift-postgres:/tmp/user_dump.sql
-        sudo podman exec -it microshift-postgres psql -d quay -U user -f /tmp/user_dump.sql >/dev/null
+        local response
+        response="$(curl -s -X POST -k  "${quay_url}/api/v1/user/initialize" \
+            --header 'Content-Type: application/json' \
+            --data '{ "username":"microshift", "password":"microshift", "email":"[email protected]", "access_token":true}')"
+        jq -e 'if .access_token then true else error(.message) end' <<< "${response}" >/dev/null
     fi
 }
 
 finalize_registry() {
     # Ensure that all the created repositories are public
     sudo podman exec -it microshift-postgres \
-        psql -d quay -U user -c 'UPDATE public.repository SET visibility_id = 1' >/dev/null
+        psql -d quay -U quayuser -c 'UPDATE public.repository SET visibility_id = 1' >/dev/null
+    # Ensure that permissions are open for the current user on the mirror registry
+    # directories and files. This is necessary to avoid 'find' command errors.
+    sudo chgrp -R "$(id -gn)" "${MIRROR_REGISTRY_DIR}"
+    sudo find "${MIRROR_REGISTRY_DIR}" -type d -exec sudo chmod a+rx '{}' \;
+    sudo find "${MIRROR_REGISTRY_DIR}" -type f -exec sudo chmod a+r  '{}' \;
 }
 
 mirror_images() {
@@ -272,3 +284,4 @@ setup_prereqs
 setup_registry
 mirror_images "${image_list_file}"
 finalize_registry
+echo "OK"