Skip to content

Commit d983fbb

Browse files
timhurskidremiotimhurskidremio
committed
DX-103340: Update jarbuild
1 parent 337eac2 commit d983fbb

1 file changed

Lines changed: 143 additions & 121 deletions

File tree

‎.github/workflows/jarbuild.yml‎

Lines changed: 143 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# under the License.
1717

1818
name: JarBuild
19-
on:
19+
on:
2020
workflow_dispatch:
2121
inputs:
2222
arrow_branch:
@@ -44,16 +44,35 @@ jobs:
4444
runs-on: ubuntu-latest
4545
timeout-minutes: 5
4646
steps:
47+
- name: Trim workflow inputs
48+
run: |
49+
echo "ARROW_BRANCH=$(echo '${{github.event.inputs.arrow_branch}}' | xargs)" >> $GITHUB_ENV
50+
echo "ARROW_REPO=$(echo '${{github.event.inputs.arrow_repo}}' | xargs)" >> $GITHUB_ENV
51+
echo "RELEASE_TAG_NAME=$(echo '${{github.event.inputs.release_tag_name}}' | xargs)" >> $GITHUB_ENV
52+
- name: Print workflow input parameters
53+
run: |
54+
echo "=========================================="
55+
echo "Workflow Input Parameters"
56+
echo "=========================================="
57+
echo "arrow_branch: ${{env.ARROW_BRANCH}}"
58+
echo "arrow_repo: ${{env.ARROW_REPO}}"
59+
echo "release_tag_name: ${{env.RELEASE_TAG_NAME}}"
60+
echo "arrow-java branch: ${{github.ref_name}}"
61+
echo ""
62+
echo "Direct Links:"
63+
echo "----------------------------------------"
64+
echo "Arrow C++ repo/branch: https://github.com/${{env.ARROW_REPO}}/tree/${{env.ARROW_BRANCH}}"
65+
echo "Arrow Java repo/branch: https://github.com/${{github.repository}}/tree/${{github.ref_name}}"
66+
echo "Release tag: https://github.com/${{github.repository}}/releases/tag/${{env.RELEASE_TAG_NAME}}"
67+
echo "=========================================="
4768
- name: Checkout
4869
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
4970
with:
5071
submodules: recursive
51-
- name: Set env
52-
run: echo "release_tag_name=$(echo $release_tag_name)" >> $GITHUB_ENV
5372
- name: Prepare for tag
5473
run: |
55-
echo "${{github.event.inputs.release_tag_name}}"
56-
ver=$(echo ${{github.event.inputs.release_tag_name}})
74+
echo "${{env.RELEASE_TAG_NAME}}"
75+
ver=$(echo ${{env.RELEASE_TAG_NAME}})
5776
version=${ver%-rc*}
5877
version=${version#v}
5978
rc=${ver#*-rc}
@@ -81,7 +100,7 @@ jobs:
81100
jni-linux:
82101
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
83102
runs-on: ${{ matrix.platform.runs_on }}
84-
timeout-minutes: 120
103+
timeout-minutes: 240
85104
needs:
86105
- source
87106
strategy:
@@ -158,17 +177,16 @@ jobs:
158177
jni-macos:
159178
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
160179
runs-on: ${{ matrix.platform.runs_on }}
161-
timeout-minutes: 45
180+
timeout-minutes: 445
162181
needs:
163182
- source
164183
strategy:
165184
fail-fast: false
166185
matrix:
167186
platform:
168-
- { runs_on: macos-13, arch: "x86_64"}
169-
- { runs_on: macos-14, arch: "aarch_64" }
187+
- { runs_on: macos-15, arch: "aarch_64" }
170188
env:
171-
MACOSX_DEPLOYMENT_TARGET: "14.0"
189+
MACOSX_DEPLOYMENT_TARGET: "15.0"
172190
steps:
173191
- name: Download source archive
174192
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -205,20 +223,41 @@ jobs:
205223
python-version: 3.12
206224
- name: Install Archery
207225
run: pip install -e arrow/dev/archery[all]
226+
- name: Checkout vcpkg
227+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
228+
with:
229+
repository: Microsoft/vcpkg
230+
path: arrow/vcpkg
231+
- name: Install vcpkg
232+
run: |
233+
cd arrow/vcpkg
234+
./bootstrap-vcpkg.sh
235+
echo "VCPKG_ROOT=${PWD}/arrow/vcpkg" >> ${GITHUB_ENV}
236+
echo "${PWD}/arrow/vcpkg" >> ${GITHUB_PATH}
237+
- name: Clean up disk space
238+
run: |
239+
echo "=== Free disk space before cleanup ==="
240+
df -h /
241+
242+
echo ""
243+
echo "=== Removing Xcode simulators ==="
244+
sudo rm -rf /Library/Developer/CoreSimulator/Caches || :
245+
echo "Removed /Library/Developer/CoreSimulator/Caches"
246+
247+
echo ""
248+
echo "=== Removing user simulator data ==="
249+
rm -rf ~/Library/Developer/CoreSimulator || :
250+
echo "Removed ~/Library/Developer/CoreSimulator"
251+
252+
echo ""
253+
echo "=== Free disk space after cleanup ==="
254+
df -h /
208255
- name: Install dependencies
209256
run: |
210-
# We want to use llvm@14 to avoid shared z3
211-
# dependency. llvm@14 doesn't depend on z3 and llvm depends
212-
# on z3. And Homebrew's z3 provides only shared library. It
213-
# doesn't provides static z3 because z3's CMake doesn't accept
214-
# building both shared and static libraries at once.
215-
# See also: Z3_BUILD_LIBZ3_SHARED in
216-
# https://github.com/Z3Prover/z3/blob/master/README-CMake.md
217-
#
218-
# If llvm is installed, Apache Arrow C++ uses llvm rather than
219-
# llvm@14 because llvm is newer than llvm@14.
220-
brew uninstall llvm || :
257+
echo "=== Free disk space at start of dependency installation ==="
258+
df -h /
221259
260+
echo ""
222261
# Ensure updating python@XXX with the "--overwrite" option.
223262
# If python@XXX is updated without "--overwrite", it causes
224263
# a conflict error. Because Python 3 installed not by
@@ -240,7 +279,13 @@ jobs:
240279
brew uninstall pkg-config@0.29.2 || :
241280
fi
242281
282+
# Install basic build tools via brew (vcpkg needs these)
283+
brew install cmake ninja pkg-config
243284
brew bundle --file=arrow/cpp/Brewfile
285+
286+
# Clean up any existing LLVM installations in favor of vcpkg.
287+
brew uninstall llvm || :
288+
244289
# We want to link aws-sdk-cpp statically but Homebrew's
245290
# aws-sdk-cpp provides only shared library. If we have
246291
# Homebrew's aws-sdk-cpp, our build mix Homebrew's
@@ -259,6 +304,24 @@ jobs:
259304
# bundled Protobuf.
260305
brew uninstall protobuf
261306
307+
echo ""
308+
echo "=== Free disk space before LLVM build ==="
309+
df -h /
310+
311+
echo ""
312+
# Use vcpkg to install LLVM.
313+
vcpkg install \
314+
--clean-after-build \
315+
--x-install-root=${VCPKG_ROOT}/installed \
316+
--x-manifest-root=arrow/ci/vcpkg \
317+
--overlay-ports=arrow/ci/vcpkg/overlay/llvm/ \
318+
--x-feature=gandiva-llvm
319+
320+
echo ""
321+
echo "=== Free disk space after LLVM build ==="
322+
df -h /
323+
324+
echo ""
262325
brew bundle --file=Brewfile
263326
- name: Prepare ccache
264327
run: |
@@ -271,103 +334,31 @@ jobs:
271334
restore-keys: jni-macos-${{ matrix.platform.arch }}-
272335
- name: Build
273336
run: |
337+
echo "=== Free disk space at start of build ==="
338+
df -h /
339+
340+
echo ""
274341
set -e
275342
# make brew Java available to CMake
276343
export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
277344
ci/scripts/jni_macos_build.sh . arrow build jni
345+
346+
echo ""
347+
echo "=== Free disk space at end of build ==="
348+
df -h /
278349
- name: Compress into single artifact to keep directory structure
279350
run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/
280351
- name: Upload artifacts
281352
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
282353
with:
283354
name: jni-macos-${{ matrix.platform.arch }}
284355
path: jni-macos-${{ matrix.platform.arch }}.tar.gz
285-
jni-windows:
286-
name: JNI ${{ matrix.platform.runs_on }} ${{ matrix.platform.arch }}
287-
runs-on: ${{ matrix.platform.runs_on }}
288-
timeout-minutes: 45
289-
needs:
290-
- source
291-
strategy:
292-
fail-fast: false
293-
matrix:
294-
platform:
295-
- runs_on: windows-2019
296-
arch: "x86_64"
297-
steps:
298-
- name: Download source archive
299-
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
300-
with:
301-
name: release-source
302-
- name: Extract source archive
303-
shell: bash
304-
run: |
305-
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
306-
# - name: Download the latest Apache Arrow C++
307-
# if: github.event_name != 'schedule'
308-
# shell: bash
309-
# run: |
310-
# ci/scripts/download_cpp.sh
311-
- name: Checkout Apache Arrow C++
312-
# if: github.event_name == 'schedule'
313-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
314-
with:
315-
repository: ${{github.event.inputs.arrow_repo}}
316-
ref: ${{github.event.inputs.arrow_branch}}
317-
path: arrow
318-
- name: Set up Java
319-
uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4.6.0
320-
with:
321-
java-version: '11'
322-
distribution: 'temurin'
323-
- name: Download Timezone Database
324-
shell: bash
325-
run: |
326-
arrow/ci/scripts/download_tz_database.sh
327-
- name: Install ccache
328-
shell: bash
329-
run: |
330-
env | sort
331-
version=4.10.2
332-
base_name="ccache-${version}-windows-x86_64"
333-
url="https://github.com/ccache/ccache/releases/download/v${version}/${base_name}.zip"
334-
curl --fail --location --remote-name "${url}"
335-
unzip "${base_name}.zip"
336-
chmod +x "${base_name}/ccache.exe"
337-
mv "${base_name}/ccache.exe" /usr/bin/
338-
rm -rf "${base_name}"{,.zip}
339-
- name: Prepare ccache
340-
shell: bash
341-
run: |
342-
echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV}
343-
- name: Cache ccache
344-
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
345-
with:
346-
path: ccache
347-
key: jni-windows-${{ matrix.platform.arch }}-${{ hashFiles('arrow/cpp/**') }}
348-
restore-keys: jni-windows-${{ matrix.platform.arch }}-
349-
- name: Build
350-
shell: cmd
351-
run: |
352-
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
353-
REM For ORC
354-
set TZDIR=/c/msys64/usr/share/zoneinfo
355-
bash -c "ci/scripts/jni_windows_build.sh . arrow build jni"
356-
- name: Compress into single artifact to keep directory structure
357-
shell: bash
358-
run: tar -cvzf jni-windows-${{ matrix.platform.arch }}.tar.gz jni/
359-
- name: Upload artifacts
360-
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
361-
with:
362-
name: jni-windows-${{ matrix.platform.arch }}
363-
path: jni-windows-${{ matrix.platform.arch }}.tar.gz
364356
binaries:
365357
name: Binaries
366358
runs-on: ubuntu-latest
367359
needs:
368360
- jni-linux
369361
- jni-macos
370-
- jni-windows
371362
steps:
372363
- name: Download artifacts
373364
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
@@ -379,9 +370,7 @@ jobs:
379370
tar -xf apache-arrow-java-*.tar.gz --strip-components=1
380371
tar -xvzf jni-linux-x86_64.tar.gz
381372
tar -xvzf jni-linux-aarch_64.tar.gz
382-
tar -xvzf jni-macos-x86_64.tar.gz
383373
tar -xvzf jni-macos-aarch_64.tar.gz
384-
tar -xvzf jni-windows-x86_64.tar.gz
385374
- name: Test that shared libraries exist
386375
run: |
387376
set -x
@@ -396,19 +385,11 @@ jobs:
396385
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
397386
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so
398387
399-
test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
400-
test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
401-
test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
402-
test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib
403-
404388
test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
405389
test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
406390
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
407391
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib
408392
409-
test -f jni/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
410-
test -f jni/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
411-
test -f jni/arrow_orc_jni/x86_64/arrow_orc_jni.dll
412393
- name: Checkout apache/arrow-testing
413394
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
414395
with:
@@ -521,30 +502,71 @@ jobs:
521502
permissions:
522503
contents: write
523504
steps:
505+
- name: Checkout arrow-java repository
506+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
507+
- name: Checkout Apache Arrow C++ repository
508+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
509+
with:
510+
repository: ${{github.event.inputs.arrow_repo}}
511+
ref: ${{github.event.inputs.arrow_branch}}
512+
path: arrow
513+
- name: Get commit IDs
514+
id: commit_ids
515+
run: |
516+
# Get short commit ID for arrow-java
517+
arrow_java_commit=$(git rev-parse --short HEAD)
518+
echo "arrow_java_commit=${arrow_java_commit}" >> $GITHUB_OUTPUT
519+
520+
# Get short commit ID for arrow
521+
cd arrow
522+
arrow_commit=$(git rev-parse --short HEAD)
523+
echo "arrow_commit=${arrow_commit}" >> $GITHUB_OUTPUT
524+
cd ..
525+
526+
# Parse version from release tag
527+
ver=$(echo ${{github.event.inputs.release_tag_name}})
528+
version=${ver%-rc*}
529+
version=${version#v}
530+
rc=${ver#*-rc}
531+
532+
# Create release name with both commit IDs
533+
release_name="${version}-${arrow_java_commit}-${arrow_commit}"
534+
release_tag="v${release_name}"
535+
echo "release_name=${release_name}" >> $GITHUB_OUTPUT
536+
echo "release_tag=${release_tag}" >> $GITHUB_OUTPUT
537+
echo "version=${version}" >> $GITHUB_OUTPUT
538+
echo "rc=${rc}" >> $GITHUB_OUTPUT
539+
540+
echo "Arrow Java commit: ${arrow_java_commit}"
541+
echo "Arrow commit: ${arrow_commit}"
542+
echo "Release tag: ${release_tag}"
524543
- name: Download release artifacts
525544
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
526545
with:
527546
pattern: release-*
528547
path: artifacts
548+
- name: Create and push tag
549+
run: |
550+
git config user.name "github-actions[bot]"
551+
git config user.email "github-actions[bot]@users.noreply.github.com"
552+
git tag -a "${{ steps.commit_ids.outputs.release_tag }}" -m "Release ${{ steps.commit_ids.outputs.release_name }} RC${{ steps.commit_ids.outputs.rc }}" -m "Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
553+
git push origin "${{ steps.commit_ids.outputs.release_tag }}"
554+
env:
555+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
529556
- name: Upload
530557
run: |
531558
# GH-499: How to create release notes?
532-
echo "${{github.event.inputs.release_tag_name}}"
533-
ver=$(echo ${{github.event.inputs.release_tag_name}})
534-
version=${ver%-rc*}
535-
version=${version#v}
536-
rc=${ver#*-rc}
537-
gh release create ${{github.event.inputs.release_tag_name}} \
538-
--generate-notes \
559+
echo "Creating release: ${{ steps.commit_ids.outputs.release_tag }}"
560+
gh release create "${{ steps.commit_ids.outputs.release_tag }}" \
561+
-n "Action URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \
539562
--prerelease \
540563
--repo ${GITHUB_REPOSITORY} \
541-
--title "Apache Arrow Java ${version} RC${rc}" \
542-
--verify-tag
564+
--title "Apache Arrow Java ${{ steps.commit_ids.outputs.version }} RC${{ steps.commit_ids.outputs.rc }} (arrow-java: ${{ steps.commit_ids.outputs.arrow_java_commit }}, arrow: ${{ steps.commit_ids.outputs.arrow_commit }})"
543565
# GitHub CLI does not respect their own rate limits
544566
# https://github.com/cli/cli/issues/9586
545567
for artifact in artifacts/*/*; do
546568
sleep 1
547-
gh release upload ${{github.event.inputs.release_tag_name}} \
569+
gh release upload "${{ steps.commit_ids.outputs.release_tag }}" \
548570
--repo ${GITHUB_REPOSITORY} \
549571
$artifact
550572
done

0 commit comments

Comments
 (0)