GH-46336: [Release][Packaging] Add support for Reproducible Builds for source archive

.github/workflows/release_candidate.yml

@@ -19,17 +19,26 @@ name: Release
 
 on:
   push:
+    branches:
+      - '**'
+      - '!dependabot/**'
     tags:
       # Trigger workflow when a tag whose name matches the pattern
       # "apache-arrow-{MAJOR}.{MINOR}.{PATCH}-rc{RC_NUM}" is pushed.
       - "apache-arrow-[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
+    paths:
+      - ".github/workflows/release_candidate.sh"
+      - "dev/release/utils-create-release-tarball.sh"
+      - "dev/release/utils-generate-checksum.sh"
+  pull_request:
+    paths:
+      - ".github/workflows/release_candidate.sh"
+      - "dev/release/utils-create-release-tarball.sh"
+      - "dev/release/utils-generate-checksum.sh"
 
 permissions:
   contents: write
 
-env:
-  GH_TOKEN: ${{ github.token }}
-
 jobs:
   publish:
     name: Publish
@@ -40,12 +49,27 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y -V gpg reprotest
       - name: Store Version and Release Candidate Number
         run: |
-          version_with_rc=${GITHUB_REF_NAME#apache-arrow-}
-          version=${version_with_rc%-rc*}
-          rc_num=${version_with_rc#${version}-rc}
-          echo "VERSION_WITH_RC=${version_with_rc}" >> ${GITHUB_ENV}
+          if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+            version_with_rc=${GITHUB_REF_NAME#apache-arrow-}
+            version=${version_with_rc%-rc*}
+            rc_num=${version_with_rc#${version}-rc}
+          else
+            version=$(grep '^set(ARROW_VERSION ' cpp/CMakeLists.txt | \
+                        grep -E -o '[0-9]+\.[0-9]+\.[0-9]+')
+            rc_num=0
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git tag \
+              -a \
+              "apache-arrow-${version}-rc${rc_num}" \
+              -m "Apache Arrow ${version} RC${rc_num}"
+          fi
           echo "VERSION=${version}" >> ${GITHUB_ENV}
           echo "RC_NUM=${rc_num}" >> ${GITHUB_ENV}
       - name: Create Release Candidate Title
@@ -57,11 +81,35 @@ jobs:
           release_notes="Release Candidate: ${VERSION} RC${RC_NUM}"
           echo "RELEASE_CANDIDATE_NOTES=${release_notes}" >> ${GITHUB_ENV}
       - name: Create Release tarball
+        env:
+          ARROW_GPG_KEY_UID: A2AC7132B5DA7C273A7A147665F4A8CA9769ECD7
+          ARROW_GPG_SECRET_KEY: ${{ secrets.ARROW_GPG_SECRET_KEY }}
         run: |
+          sudo reprotest \
+            "dev/release/utils-create-release-tarball.sh ${VERSION} ${RC_NUM}" \
+            apache-arrow-${VERSION}.tar.gz
           dev/release/utils-create-release-tarball.sh ${VERSION} ${RC_NUM}
           echo "RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz" >> ${GITHUB_ENV}
           dev/release/utils-generate-checksum.sh "apache-arrow-${VERSION}.tar.gz"
+          if [ -n "${ARROW_GPG_SECRET_KEY}" ]; then
+            echo "${ARROW_GPG_SECRET_KEY}" | gpg --import
+            gpg \
+              --armor \
+              --detach-sign \
+              --local-user "${ARROW_GPG_SECRET_KEY}" \
+              --output "${RELEASE_TARBALL}.asc" \
+              "${RELEASE_TARBALL}"
+          fi
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: release-candidate
+          path: ${{ env.RELEASE_TARBALL }}*
       - name: Create GitHub Release
+        if: |
+          github.ref_type == 'tag'
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           gh release create ${GITHUB_REF_NAME} \
             --verify-tag \

dev/release/02-source.sh

@@ -75,45 +75,21 @@ if [ ${SOURCE_RAT} -gt 0 ]; then
 fi
 
 if [ ${SOURCE_UPLOAD} -gt 0 ]; then
-  rm -rf signed-artifacts
-  mkdir -p signed-artifacts
-
-  # sign the artifacts
-  for artifact in artifacts/*; do
-    case "${artifact}" in
-      *.sha256|*.sha512)
-        continue
-        ;;
-    esac
-    gpg \
-      --armor \
-      --detach-sig \
-      --output signed-artifacts/$(basename ${artifact}).asc \
-      ${artifact}
-  done
-
-  # Upload signed tarballs to GitHub Release
-  gh release upload ${tag} \
-     --repo "${GITHUB_REPOSITORY}" \
-     signed-artifacts/*
-
   # check out the arrow RC folder
   svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow tmp
 
   # add the release candidate for the tag
   mkdir -p tmp/${tag}
 
-  # copy the rc tarball into the tmp dir
+  # copy the release candidate tarball and related files into the tmp dir
   cp artifacts/${tarball}* tmp/${tag}
-  cp signed-artifacts/${tarball}.asc tmp/${tag}
 
   # commit to svn
   svn add tmp/${tag}
   svn ci -m "Apache Arrow ${version} RC${rc}" tmp/${tag}
 
   # clean up
   rm -rf artifacts
-  rm -rf signed-artifacts
   rm -rf tmp
 
   echo "Success! The release candidate is available here:"

dev/release/utils-create-release-tarball.sh

@@ -18,6 +18,12 @@
 # under the License.
 #
 
+# This must generate reproducible source archive. For reproducible
+# source archive, we must use the same timestamp, user, group and so
+# on.
+#
+# See also https://reproducible-builds.org/ for Reproducible Builds.
+
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
 
@@ -46,18 +52,70 @@ rm -rf ${root_folder}
 # Resolve symbolic and hard links
 rm -rf ${root_folder}.tmp
 mv ${root_folder} ${root_folder}.tmp
-cp -R -L ${root_folder}.tmp ${root_folder}
+cp -a -L ${root_folder}.tmp ${root_folder}
 rm -rf ${root_folder}.tmp
 
-# Create a dummy .git/ directory to download the source files from GitHub with Source Link in C#.
+# Create a dummy .git/ directory to download the source files from
+# GitHub with Source Link in C#. See the followings for more Source
+# Link support and why we need a dummy .git directory:
+# * https://github.com/apache/arrow/issues/21580
+# * https://github.com/apache/arrow/pull/4488
+#
+# We need to set constant timestamp for a dummy .git/ directory for
+# Reproducible Builds. We use mtime of csharp/ for it. If
+# SOURCE_DATE_EPOCH is defined, this mtime is overwritten when tar
+# file is created.
+if stat --help > /dev/null 2>&1; then
+  # GNU stat
+  #
+  # touch accepts YYYYMMDDhhmm.ss format but GNU stat doesn't support
+  # for outputting with the format. So we use date for it.
+  csharp_mtime=$(date --date="$(stat --format="%y" csharp)" +%Y%m%d%H%M.%S)
+else
+  # BSD stat
+  csharp_mtime=$(stat -f %Sm -t %Y%m%d%H%M.%S csharp)
+fi
 dummy_git=${root_folder}/csharp/dummy.git
 mkdir ${dummy_git}
 pushd ${dummy_git}
 echo ${release_hash} > HEAD
 echo "[remote \"origin\"] url = https://github.com/${GITHUB_REPOSITORY:-apache/arrow}.git" >> config
 mkdir objects refs
+find . -exec touch -t "${csharp_mtime}" '{}' ';'
 popd
+touch -t "${csharp_mtime}" ${root_folder}/csharp
 
-# Create new tarball from modified source directory
-tar czf ${tarball} ${root_folder}
+# Create new tarball from modified source directory.
+#
+# We need to strip unreproducible information. See also:
+# https://reproducible-builds.org/docs/stripping-unreproducible-information/
+#
+# We need GNU tar for Reproducible Builds. We want to use the same
+# owner, group, mode, file order for Reproducible Builds.
+# See also: https://reproducible-builds.org/docs/archives/
+#
+# gzip --no-name is for omitting timestamp in .gz. It's also for
+# Reproducible Builds.
+if type gtar > /dev/null 2>&1; then
+  gtar=gtar
+else
+  gtar=tar
+fi
+gtar_options=(
+  --group=0 \
+  --mode=a=rX,u+w \
+  --numeric-owner \
+  --owner=0 \
+  --sort=name \
+)
+if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then
+  gtar_options+=(--mtime="$(date +%Y-%m-%dT%H:%M:%S --date=@${SOURCE_DATE_EPOCH})")
+fi
+set -x
+${gtar} \
+  "${gtar_options[@]}" \
+  -cf \
+  - \
+  ${root_folder} | \
+    gzip --no-name > ${tarball}
 rm -rf ${root_folder}

dev/release/verify-release-candidate.sh

@@ -74,6 +74,8 @@ esac
 SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
 ARROW_DIR="$(cd "${SOURCE_DIR}/../.." && pwd)"
 
+: ${GITHUB_REPOSITORY:=apache/arrow}
+
 show_header() {
   echo ""
   printf '=%.0s' $(seq ${#1}); printf '\n'
@@ -176,7 +178,7 @@ test_binary() {
 
   ${PYTHON:-python3} $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
          --dest=${download_dir} \
-         --repository=${GITHUB_REPOSITORY:-apache/arrow} \
+         --repository=${GITHUB_REPOSITORY} \
          --tag="apache-arrow-$VERSION-rc$RC_NUMBER"
 
   verify_dir_artifact_signatures ${download_dir}
@@ -787,6 +789,14 @@ ensure_source_directory() {
     if [ ! -d "${ARROW_SOURCE_DIR}" ]; then
       pushd $ARROW_TMPDIR
       fetch_archive ${dist_name}
+      git clone https://github.com/${GITHUB_REPOSITORY}.git arrow
+      pushd arrow
+      dev/release/utils-create-release-tarball.sh ${VERSION} ${RC_NUMBER}
+      if ! cmp ${dist_name}.tar.gz ../${dist_name}.tar.gz; then
+        echo "Source archive isn't reproducible"
+        return 1
+      fi
+      popd
       tar xf ${dist_name}.tar.gz
       popd
     fi
@@ -988,7 +998,7 @@ test_wheels() {
       --package_type python \
       --regex=${filter_regex} \
       --dest=${download_dir} \
-      --repository=${GITHUB_REPOSITORY:-apache/arrow} \
+      --repository=${GITHUB_REPOSITORY} \
       --tag="apache-arrow-$VERSION-rc$RC_NUMBER"
 
     verify_dir_artifact_signatures ${download_dir}