Skip to content

Commit 39770de

Browse files
koukou
authored
GH-50170: [CI][Packaging][Linux] Fix cache (#50185)
### Rationale for this change We want to reduce CI time as much as possible for developer experience and avoiding consuming shared resources. ### What changes are included in this PR? * Use `apache/infrastructure-actions/stash` instead of `actions/cache` to use larger cache storage * Enable ccache * Increase ccache size because 500MB is small for our case * Fix Docker image cache * Disable RapidJSON's ccache detection that causes needless `ccache` call such as `ccache ccache cc ...` * Use Ninja instead of Make ### Are these changes tested? Yes. CI time is reduced to 20min-60min from 60min-120min. ### Are there any user-facing changes? No. * GitHub Issue: #50170 Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 4bf9a16 commit 39770de

12 files changed

Lines changed: 58 additions & 20 deletions

File tree

‎.github/workflows/package_linux.yml‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,10 @@ jobs:
7676

7777
package:
7878
permissions:
79-
# Upload to GitHub Release
79+
# Upload artifacts to GitHub Release
8080
contents: write
81+
# Upload cached Docker images to GitHub Packages
82+
packages: write
8183
name: ${{ matrix.id }}
8284
runs-on: ${{ contains(matrix.id, 'amd64') && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
8385
needs: check-labels
@@ -186,12 +188,11 @@ jobs:
186188
version="${GITHUB_REF_NAME#apache-arrow-}"
187189
echo "ARROW_VERSION=${version}" >> "${GITHUB_ENV}"
188190
fi
189-
- name: Cache ccache
190-
uses: actions/cache@v5
191+
- name: Restore ccache
192+
uses: apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
191193
with:
192-
path: ${{ env.BUILD_DIR }}/ccache
193-
key: package-linux-${{ matrix.id }}-${{ hashFiles('cpp/**', 'c_glib/**') }}
194-
restore-keys: package-linux-${{ matrix.id }}-
194+
path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
195+
key: package-linux-${{ matrix.id }}
195196
- name: Install dependencies
196197
run: |
197198
sudo apt update
@@ -249,16 +250,25 @@ jobs:
249250
env:
250251
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
251252
run: |
252-
rake -C dev/tasks/linux-packages docker:pull || :
253+
rake -C dev/tasks/linux-packages/apache-arrow docker:pull:${TARGET} || :
253254
rake -C dev/tasks/linux-packages ${TASK_NAMESPACE}:build
255+
- name: Save ccache
256+
if: >-
257+
!cancelled()
258+
continue-on-error: true
259+
uses: apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
260+
with:
261+
path: ${{ env.BUILD_DIR }}/${{ env.TARGET }}
262+
key: package-linux-${{ matrix.id }}
263+
include-hidden-files: true
254264
- name: Docker Push
255265
continue-on-error: true
256266
if: >-
257267
success() &&
258268
github.event_name == 'push' &&
259269
github.ref_name == 'main'
260270
run: |
261-
rake -C dev/tasks/linux-packages docker:push
271+
rake -C dev/tasks/linux-packages/apache-arrow docker:push:${TARGET}
262272
- name: Build artifact tarball
263273
run: |
264274
mkdir -p "${DISTRIBUTION}"
@@ -339,7 +349,6 @@ jobs:
339349
env:
340350
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
341351
run: |
342-
rake -C dev/tasks/linux-packages docker:pull || :
343352
# Validate reproducibility. Reprotest runs the build twice
344353
# inside its own tempdir and doesn't copy artifacts back,
345354
# so this is purely a verification step.

‎cpp/cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2791,6 +2791,7 @@ function(build_rapidjson)
27912791
URL ${RAPIDJSON_SOURCE_URL}
27922792
URL_HASH "SHA256=${ARROW_RAPIDJSON_BUILD_SHA256_CHECKSUM}")
27932793
prepare_fetchcontent()
2794+
set(CCACHE_FOUND OFF)
27942795
set(LIB_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/lib")
27952796
set(RAPIDJSON_BUILD_DOC OFF)
27962797
set(RAPIDJSON_BUILD_EXAMPLES OFF)

‎dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ RUN \
3838
apt update ${quiet} && \
3939
apt install -y -V ${quiet} \
4040
build-essential \
41+
ccache \
4142
clang \
4243
cmake \
4344
debhelper \
@@ -81,6 +82,8 @@ RUN \
8182
tzdata \
8283
valac \
8384
zlib1g-dev && \
85+
# Ensure updating symlinks in /usr/lib/ccache/
86+
update-ccache-symlinks && \
8487
if apt list | grep '^nvidia-cuda-toolkit/'; then \
8588
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
8689
fi && \

‎dev/tasks/linux-packages/apache-arrow/apt/debian-forky/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ RUN \
3939
apt install -y -V ${quiet} \
4040
base-files \
4141
build-essential \
42+
ccache \
4243
clang \
4344
cmake \
4445
debhelper \
@@ -81,6 +82,8 @@ RUN \
8182
tzdata \
8283
valac \
8384
zlib1g-dev && \
85+
# Ensure updating symlinks in /usr/lib/ccache/
86+
update-ccache-symlinks && \
8487
if apt list | grep '^nvidia-cuda-toolkit/'; then \
8588
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
8689
fi && \

‎dev/tasks/linux-packages/apache-arrow/apt/debian-trixie/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ RUN \
8282
tzdata \
8383
valac \
8484
zlib1g-dev && \
85+
# Ensure updating symlinks in /usr/lib/ccache/
86+
update-ccache-symlinks && \
8587
if apt list | grep '^nvidia-cuda-toolkit/'; then \
8688
apt install -y -V ${quiet} nvidia-cuda-toolkit && \
8789
# GH-44358: Workaround for non-existent path error

‎dev/tasks/linux-packages/apache-arrow/apt/ubuntu-jammy/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ RUN \
3232
apt update ${quiet} && \
3333
apt install -y -V ${quiet} \
3434
build-essential \
35+
ccache \
3536
clang \
3637
cmake \
3738
curl \
@@ -76,6 +77,8 @@ RUN \
7677
tzdata \
7778
valac \
7879
zlib1g-dev && \
80+
# Ensure updating symlinks in /usr/lib/ccache/
81+
update-ccache-symlinks && \
7982
if apt list | grep -q '^libcuda1'; then \
8083
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
8184
else \

‎dev/tasks/linux-packages/apache-arrow/apt/ubuntu-noble/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ RUN \
3232
apt update ${quiet} && \
3333
apt install -y -V ${quiet} \
3434
build-essential \
35+
ccache \
3536
clang \
3637
clang-tools \
3738
cmake \
@@ -78,6 +79,8 @@ RUN \
7879
tzdata \
7980
valac \
8081
zlib1g-dev && \
82+
# Ensure updating symlinks in /usr/lib/ccache/
83+
update-ccache-symlinks && \
8184
if apt list | grep -q '^libcuda'; then \
8285
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
8386
else \

‎dev/tasks/linux-packages/apache-arrow/apt/ubuntu-resolute/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ RUN \
3232
apt update ${quiet} && \
3333
apt install -y -V ${quiet} \
3434
build-essential \
35+
ccache \
3536
clang \
3637
clang-tools \
3738
cmake \
@@ -76,6 +77,8 @@ RUN \
7677
tzdata \
7778
valac \
7879
zlib1g-dev && \
80+
# Ensure updating symlinks in /usr/lib/ccache/
81+
update-ccache-symlinks && \
7982
if apt list | grep -q '^libcuda'; then \
8083
apt install -y -V ${quiet} nvidia-cuda-toolkit; \
8184
else \

‎dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2023/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,14 @@ ARG DEBUG
2323
RUN \
2424
quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \
2525
dnf update -y ${quiet} && \
26+
dnf install -y ${quiet} spal-release && \
2627
dnf install -y ${quiet} \
2728
bison \
2829
boost-devel \
2930
brotli-devel \
3031
bzip2-devel \
3132
c-ares-devel \
33+
ccache \
3234
clang \
3335
cmake \
3436
curl-devel \

‎dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,13 @@
3737
%if %{_rhel} >= 9 || %{_amzn} >= 2023
3838
%define arrow_cmake_builddir %{__cmake_builddir}
3939
%define arrow_cmake %cmake
40-
%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
40+
%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
4141
%define arrow_cmake_install %cmake_install
4242
%else
4343
%define arrow_cmake_builddir build
4444
%define arrow_cmake %cmake3 -S . -B %{arrow_cmake_builddir}
45-
%define arrow_cmake_build make -C %{arrow_cmake_builddir} %{?_smp_mflags}
46-
%define arrow_cmake_install DESTDIR="%{buildroot}" make -C %{arrow_cmake_builddir} install
45+
%define arrow_cmake_build cmake --build %{arrow_cmake_builddir}
46+
%define arrow_cmake_install DESTDIR="%{buildroot}" cmake --install %{arrow_cmake_builddir}
4747
%endif
4848

4949
%define use_bundled_absl (%{_rhel} < 10)
@@ -131,6 +131,11 @@ Apache Arrow is a data processing library for analysis.
131131

132132
%build
133133
cpp_build_type=release
134+
if [ -n "${X_SCLS:-}" ]; then
135+
ARROW_USE_CCACHE=ON
136+
else
137+
ARROW_USE_CCACHE=OFF
138+
fi
134139
cd cpp
135140
%arrow_cmake \
136141
-DARROW_AZURE=ON \
@@ -154,7 +159,7 @@ cd cpp
154159
%if %{use_s3}
155160
-DARROW_S3=ON \
156161
%endif
157-
-DARROW_USE_CCACHE=OFF \
162+
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
158163
-DARROW_WITH_BROTLI=ON \
159164
-DARROW_WITH_BZ2=ON \
160165
-DARROW_WITH_LZ4=ON \
@@ -170,7 +175,7 @@ cd cpp
170175
%if %{use_bundled_nlohmann_json}
171176
-Dnlohmann_json_SOURCE=BUNDLED \
172177
%endif
173-
-G"Unix Makefiles"
178+
-GNinja
174179
%arrow_cmake_build
175180
cd -
176181

0 commit comments

Comments
 (0)