Merge pull request #16333 from MinaProtocol/dkijania/port_new_deb_s3_dev

[DEV] Use new version of deb-s3 for validating job publishing

buildkite/src/Constants/ContainerImages.dhall

@@ -4,11 +4,11 @@
 -- NOTE: minaToolchainBookworm is also used for building Ubuntu Jammy packages in CI
 { toolchainBase = "codaprotocol/ci-toolchain-base:v3"
 , minaToolchainBullseye =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d"
 , minaToolchainBookworm =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d"
 , minaToolchain =
-    "gcr.io/o1labs-192920/mina-toolchain@sha256:1e4b429fdf6a06e88abe9b4c9d54a80a3e818011a94806d070433a95d6af1229"
+    "gcr.io/o1labs-192920/mina-toolchain@sha256:de33ba62503099b76dbdd936a65ebc50128238deb88e573cd6fee49ec403ae2d"
 , elixirToolchain = "elixir:1.10-alpine"
 , nodeToolchain = "node:14.13.1-stretch-slim"
 , ubuntu2004 = "ubuntu:20.04"

dockerfiles/stages/3-toolchain

@@ -8,7 +8,7 @@ ARG deb_codename=focal
 
 ARG DOCKER_VERSION=19.03.4
 ARG TERRAFORM_VERSION=0.14.11
-ARG DEBS3_VERSION=0.11.6
+ARG DEBS3_VERSION=0.11.7
 ARG DHALL_VERSION=1.41.1
 ARG DHALL_JSON_VERSION=1.7.10
 ARG DHALL_BASH_VERSION=1.0.40

scripts/debian/publish.sh

@@ -34,8 +34,8 @@ if [[ -z "$DEB_CODENAME" ]]; then usage "Codename is not set!"; fi;
 if [[ -z "$DEB_RELEASE" ]]; then usage "Release is not set!"; fi;
 
 
-BUCKET_ARG='--bucket packages.o1test.net'
-S3_REGION_ARG='--s3-region=us-west-2'
+BUCKET_ARG="--bucket=packages.o1test.net"
+S3_REGION_ARG="--s3-region=us-west-2"
 # utility for publishing deb repo with commons options
 # deb-s3 https://github.com/krobertson/deb-s3
 #NOTE: Do not remove --lock flag otherwise racing deb uploads may overwrite the registry and some files will be lost. If a build fails with the following error, delete the lock file https://packages.o1test.net/dists/unstable/main/binary-/lockfile and rebuild
@@ -52,43 +52,54 @@ DEBS3_UPLOAD="deb-s3 upload $BUCKET_ARG $S3_REGION_ARG \
 echo "Publishing debs: ${DEB_NAMES} to Release: ${DEB_RELEASE} and Codename: ${DEB_CODENAME}"
 # Upload the deb files to s3.
 # If this fails, attempt to remove the lockfile and retry.
-for _i in {1..10}; do (
+for _ in {1..10}; do (
   ${DEBS3_UPLOAD} \
     --component "${DEB_RELEASE}" \
     --codename "${DEB_CODENAME}" \
     "${DEB_NAMES}"
 ) && break || scripts/debian/clear-s3-lockfile.sh; done
 
+debs=()
+
 for deb in $DEB_NAMES
 do
-
-  DEBS3_SHOW="deb-s3 show $BUCKET_ARG $S3_REGION_ARG"
-
   # extracting name from debian package path. E.g:
   # _build/mina-archive_3.0.1-develop-a2a872a.deb -> mina-archive
   deb=$(basename "$deb")
   deb="${deb%_*}"
+  debs+=("$deb")
+done
 
-  for i in {1..10}; do 
+function join_by { local IFS="$1"; shift; echo "$*"; }
 
-    set +e
-    ${DEBS3_SHOW} "$deb" "${DEB_VERSION}" "${ARCH}" -c "${DEB_CODENAME}" -m "${DEB_RELEASE}"
-    LAST_VERIFY_STATUS=$?
-    set -eo pipefail
+tries=10
+counter=0
 
-    if [[ $LAST_VERIFY_STATUS == 0 ]]; then
-        echo "succesfully validated that package is uploaded to deb-s3"
-        break
-    fi
-
-    sleep 60
-    i=$((i+1)) 
+while (( ${#debs[@]} ))
+do
+  join=$(join_by " " "${debs[@]}")
+
+  IFS=$'\n'
+  output=$(deb-s3 exist $BUCKET_ARG $S3_REGION_ARG "$join" $DEB_VERSION $ARCH -c $DEB_CODENAME -m $DEB_RELEASE)
+  debs=()
+  for item in $output; do
+     if [[ $item == *"Missing" ]]; then
+      key=$(echo "$item" | awk '{print $1}')
+      debs+=("$key")
+     fi
   done
 
-  if [[ $LAST_VERIFY_STATUS != 0 ]]; then
-    echo "Cannot locate '$deb' in debian repo. failing job..."
+  if [ ${#debs[@]} -eq 0 ]; then
+    echo "All debians are correctly published to our debian repository"
+    exit 0
+  fi
+
+  counter=$((counter+1))
+  if [[ $((counter)) == $((tries)) ]]; then
+    echo "Error: Some Debians are still not correctly published : "$(join_by " " "${debs[@]}")
     echo "You may still try to rerun job as debian repository is known from imperfect performance"
     exit 1
-  fi
+  fi 
 
-done
+  sleep 60
+done