GH-50170: [CI][Packaging][Linux] Fix cache

Author: kou

.github/workflows/package_linux.yml

@@ -76,8 +76,10 @@ jobs:
 
   package:
     permissions:
-      # Upload to GitHub Release
+      # Upload artifacts to GitHub Release
       contents: write
+      # Upload cached Docker images to GitHub Packages
+      packages: write
     name: ${{ matrix.id }}
     runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
     needs: check-labels
@@ -186,12 +188,11 @@ jobs:
             version="${GITHUB_REF_NAME#apache-arrow-}"
             echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}"
           fi
-      - name: Cache ccache
-        uses: actions/cache@v5
+      - name: Restore ccache
+        uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
         with:
-          path: ${{ env.BUILD_DIR }}/ccache
-          key:  package-linux-${{ matrix.id }}-${{ hashFiles('cpp/**', 'c_glib/**') }}
-          restore-keys: package-linux-${{ matrix.id }}-
+          path: ${{ env.BUILD_DIR }}
+          key: package-linux-${{ matrix.id }}
       - name: Install dependencies
         run: |
           sudo apt update
@@ -249,16 +250,25 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          rake -C dev/tasks/linux-packages docker:pull || :
+          rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET} || :
           rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build
+      - name: Save ccache
+        if: >-
+          !cancelled()
+        continue-on-error: true
+        uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+        with:
+          path: ${{ env.BUILD_DIR }}
+          key: package-linux-${{ matrix.id }}
+          include-hidden-files: true
       - name: Docker Push
         continue-on-error: true
         if: >-
           success() &&
           github.event_name == 'push' &&
           github.ref_name == 'main'
         run: |
-          rake -C dev/tasks/linux-packages docker:push
+          rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET}
       - name: Build artifact tarball
         run: |
           mkdir -p "${DISTRIBUTION}"
@@ -339,7 +349,6 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          rake -C dev/tasks/linux-packages docker:pull || :
           # Validate reproducibility. Reprotest runs the build twice
           # inside its own tempdir and doesn't copy artifacts back,
           # so this is purely a verification step.

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2791,6 +2791,7 @@ function(build_rapidjson)
                        URL ${RAPIDJSON_SOURCE_URL}
                        URL_HASH "SHA256=${ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM}")
   prepare_fetchcontent()
+  set(CCACHE_FOUND OFF)
   set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib")
   set(RAPIDJSON_BUILD_DOC OFF)
   set(RAPIDJSON_BUILD_EXAMPLES OFF)

dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile

@@ -38,6 +38,7 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    ccache \
     clang \
     cmake \
     debhelper \

dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile

@@ -39,6 +39,7 @@ RUN \
   apt install -y -V ${quiet} \
     base-files \
     build-essential \
+    ccache \
     clang \
     cmake \
     debhelper \

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile

@@ -32,6 +32,7 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    ccache \
     clang \
     cmake \
     curl \

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile

@@ -32,6 +32,7 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    ccache \
     clang \
     clang-tools \
     cmake \

dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile

@@ -32,6 +32,7 @@ RUN \
   apt update ${quiet} && \
   apt install -y -V ${quiet} \
     build-essential \
+    ccache \
     clang \
     clang-tools \
     cmake \

dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile

@@ -23,12 +23,14 @@ ARG DEBUG
 RUN \
   quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \
   dnf update -y ${quiet} && \
+  dnf install -y ${quiet} spal-release && \
   dnf install -y ${quiet} \
     bison \
     boost-devel \
     brotli-devel \
     bzip2-devel \
     c-ares-devel \
+    ccache \
     clang \
     cmake \
     curl-devel \

dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in

@@ -37,13 +37,13 @@
 %if %{_rhel} >= 9 || %{_amzn} >= 2023
 %define arrow_cmake_builddir %{__cmake_builddir}
 %define arrow_cmake %cmake
-%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
+%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
 %define arrow_cmake_install %cmake_install
 %else
 %define arrow_cmake_builddir build
 %define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir}
-%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
-%define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install
+%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
+%define arrow_cmake_install DESTDIR="%{buildroot}" cmake --install %{arrow_cmake_builddir}
 %endif
 
 %define use_bundled_absl (%{_rhel} < 10)
@@ -131,6 +131,11 @@ Apache Arrow is a data processing library for analysis.
 
 %build
 cpp_build_type=release
+if [ -n "${X_SCLS:-}" ]; then
+  ARROW_USE_CCACHE=ON
+else
+  ARROW_USE_CCACHE=OFF
+fi
 cd cpp
 %arrow_cmake \
   -DARROW_AZURE=ON \
@@ -154,7 +159,7 @@ cd cpp
 %if %{use_s3}
   -DARROW_S3=ON \
 %endif
-  -DARROW_USE_CCACHE=OFF \
+  -DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
   -DARROW_WITH_BROTLI=ON \
   -DARROW_WITH_BZ2=ON \
   -DARROW_WITH_LZ4=ON \
@@ -170,7 +175,7 @@ cd cpp
 %if %{use_bundled_nlohmann_json}
   -Dnlohmann_json_SOURCE=BUNDLED \
 %endif
-  -G"Unix Makefiles"
+  -GNinja
 %arrow_cmake_build
 cd -
 

dev/tasks/linux-packages/apt/build.sh

@@ -71,16 +71,18 @@ find . -not -path ./ccache -a -not -path "./ccache/*" -delete
 if which ccache > /dev/null 2>&1; then
   export CCACHE_COMPILERCHECK=content
   export CCACHE_COMPRESS=1
-  export CCACHE_COMPRESSLEVEL=6
   export CCACHE_DIR="${PWD}/ccache"
-  export CCACHE_MAXSIZE=500M
+  export CCACHE_LOGFILE="${PWD}/ccache.log"
+  export CCACHE_MAXSIZE=3GiB
+  rm -f "${CCACHE_LOGFILE}"
+  ccache --zero-stats
   ccache --show-stats --verbose || :
   debuild_options+=(-eCCACHE_COMPILERCHECK)
   debuild_options+=(-eCCACHE_COMPRESS)
-  debuild_options+=(-eCCACHE_COMPRESSLEVEL)
   debuild_options+=(-eCCACHE_DIR)
+  debuild_options+=(-eCCACHE_LOGFILE)
   debuild_options+=(-eCCACHE_MAXSIZE)
-  if [ -d /usr/lib/ccache ] ;then
+  if [ -d /usr/lib/ccache ]; then
     debuild_options+=(--prepend-path=/usr/lib/ccache)
   fi
 fi