From c319b87e275dd951548fec9b8791d1bf127a21da Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 10:42:19 -0800 Subject: [PATCH 01/13] upload artifacts --- .github/workflows/tests.yml | 18 +++++++++++++++++- pom.xml | 4 ++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e2aaff1..8558039 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: os: [ubuntu-latest] java-distribution: [adopt] java-version: [17] - dockcross-tag: ["20220906-e88a3ce"] + dockcross-tag: ["20220906-e88a3ce", "20230116-670f7f7"] steps: - uses: actions/checkout@v2.1.1 @@ -77,6 +77,15 @@ jobs: - name: Tests run: mvn "-Dh3.system.prune=true" "-Dh3.dockcross.tag=${{ matrix.dockcross-tag }}" -B -V clean test + - uses: actions/upload-artifact@v3 + name: Upload artifacts + with: + name: docker-built-shared-objects + path: | + src/main/resources/*/*.so + src/main/resources/*/*.dll + if-no-files-found: error + tests-no-docker: name: Java (No Docker) ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -109,6 +118,13 @@ jobs: - name: Tests run: mvn -B -V clean test site + - uses: actions/upload-artifact@v3 + name: Upload Mac OS Artifacts + with: + name: macos-built-shared-objects + path: src/main/resources/*/*.dylib + if-no-files-found: error + tests-coverage: name: Java (Coverage) ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} diff --git a/pom.xml b/pom.xml index 6b82824..56e2c38 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ com.uber h3 jar - 4.0.3-SNAPSHOT + 4.1.0 h3 https://github.com/uber/h3-java Java bindings for H3, a hierarchical hexagonal geospatial indexing system. @@ -40,7 +40,7 @@ scm:git:git://github.com/uber/h3-java.git scm:git:ssh://git@github.com/uber/h3-java.git http://github.com/uber/h3-java/tree/master - HEAD + v4.1.0 From 878c2028af1520cd52804c28b8693c3d1d45a24f Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 11:09:24 -0800 Subject: [PATCH 02/13] fix upload running on windows --- .github/workflows/tests.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8558039..adccd74 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,6 +45,16 @@ jobs: - name: Format check for C run: git diff --exit-code + - uses: actions/upload-artifact@v3 + name: Upload artifacts + if: ${{ matrix.java-version == 8 }} + with: + name: docker-built-shared-objects + path: | + src/main/resources/*/*.so + src/main/resources/*/*.dll + if-no-files-found: error + tests-new-dockcross: name: Dockcross ${{ matrix.dockcross-tag }} Java ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -77,15 +87,6 @@ jobs: - name: Tests run: mvn "-Dh3.system.prune=true" "-Dh3.dockcross.tag=${{ matrix.dockcross-tag }}" -B -V clean test - - uses: actions/upload-artifact@v3 - name: Upload artifacts - with: - name: docker-built-shared-objects - path: | - src/main/resources/*/*.so - src/main/resources/*/*.dll - if-no-files-found: error - tests-no-docker: name: Java (No Docker) ${{ matrix.java-version }} ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -120,6 +121,7 @@ jobs: - uses: actions/upload-artifact@v3 name: Upload Mac OS Artifacts + if: ${{ matrix.os == 'macos-latest' && matrix.java-version == 8 }} with: name: macos-built-shared-objects path: src/main/resources/*/*.dylib From 769fed998dc9185c9c2f6a21d97d1af66064789a Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 12:11:39 -0800 Subject: [PATCH 03/13] add download from github script --- docs/releasing.md | 8 ++- pom.xml | 2 + src/main/c/h3-java/build-h3.sh | 29 ++++++---- src/main/c/h3-java/pull-from-github.sh | 78 ++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 12 deletions(-) create mode 100755 src/main/c/h3-java/pull-from-github.sh diff --git a/docs/releasing.md b/docs/releasing.md index 3f7b749..04f1ac3 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,13 +8,17 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that 1. Ensure you are on branch `master` and that the library is building correctly. 2. Update [CHANGELOG.md](../CHANGELOG.md) to have the correct date and new version number, update [README.md](../README.md) to have the correct version numbers, and commit. -3. `mvn release:prepare` Use the new version number when prompted. -4. `mvn release:perform` +3. `mvn release:prepare -Dh3.github.artifacts=true` Use the new version number when prompted. +4. `mvn release:perform -Dh3.github.artifacts=true` 5. If this looks good, close and release the build in [Sonatype Nexus Manager](https://oss.sonatype.org/). 6. Update `CHANGELOG.md` to have an Unreleased section, and commit. The release is now done and development can resume from this point. ## Troubleshooting +### Github artifacts + +You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope. + ### gpg: signing failed: Inappropriate ioctl for device Per [StackOverflow](https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven), run the following before `mvn release:perform`: diff --git a/pom.xml b/pom.xml index 56e2c38..63287a6 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ true false 20210624-de7b1b0 + false @@ -268,6 +269,7 @@ ${h3.use.docker} ${h3.system.prune} ${h3.dockcross.tag} + ${h3.github.artifacts} diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 2f9fd93..443617d 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -15,16 +15,19 @@ # limitations under the License. # -# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] -# git-remote - The git remote to pull from. An existing cloned repository -# will not be deleted if a new remote is specified. -# git-ref - Specific git ref of H3 to build. -# use-docker - "true" to perform cross compilation via Docker, "false" to -# skip that step. -# system-prune - If use-docker is true and this argument is true, Docker -# system prune will be run after each step -# (i.e. for disk space constrained environments like CI) -# dockcross-tag - Tag name for dockcross +# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] +# git-remote - The git remote to pull from. An existing cloned repository +# will not be deleted if a new remote is specified. +# git-ref - Specific git ref of H3 to build. +# use-docker - "true" to perform cross compilation via Docker, "false" to +# skip that step. +# system-prune - If use-docker is true and this argument is true, Docker +# system prune will be run after each step +# (i.e. for disk space constrained environments like CI) +# dockcross-tag - Tag name for dockcross +# github-artifacts - When set, all build artifacts are retrieved from Github +# Actions artifacts rather than built locally (overrides +# all other settings.) # # This script downloads H3, builds H3 and the H3-Java native library, and # cross compiles via Docker. @@ -39,6 +42,12 @@ GIT_REVISION=$2 USE_DOCKER=$3 SYSTEM_PRUNE=$4 DOCKCROSS_TAG=$5 +PULL_FROM_GITHUB=$6 + +if $PULL_FROM_GITHUB; then + ./pull-from-github.sh + exit 0 +fi echo Downloading H3 from "$GIT_REMOTE" diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh new file mode 100755 index 0000000..60496d0 --- /dev/null +++ b/src/main/c/h3-java/pull-from-github.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Copyright 2023 Uber Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Retrieves built artifacts from Github. You must install the Github CLI +# and authenticate with a personal access token (classic) with workflows +# scope to use this. https://cli.github.com/ +# +# This script expects to be run from the project's base directory (where +# pom.xml is) as part of the Maven build process. + +set -ex + +GIT_REVISION=$(git rev-parse HEAD) +EXTRACT_TO=src/main/resources + +echo downloading artifacts for $GIT_REVISION + +mkdir -p target +pushd target + +TO_DOWNLOAD=$(gh api \ + -H "Accept: application/vnd.github+json" \ + /repos/{owner}/{repo}/actions/artifacts \ + | jq ".artifacts[] | select(.workflow_run.head_sha == \"$GIT_REVISION\")") + +echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do + ARTIFACT_NAME=$(echo $artifactline | jq -r .name) + ARTIFACT_ID=$(echo $artifactline | jq .id) + echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID" + gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip" + unzip "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" +done + +popd + +echo Checking that expected images are present: + +for image in android-arm android-arm64 linux-arm64 linux-armv5 linux-armv7 linux-s390x \ + linux-ppc64le linux-x64 linux-x86; do + if [ -f "$EXTRACT_TO/$image/libh3-java.so" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done + +for image in darwin-x64 darwin-arm64; do + if [ -f "$EXTRACT_TO/$image/libh3-java.dylib" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done + +for image in windows-x64 windows-x86; do + if [ -f "$EXTRACT_TO/$image/libh3-java.dll" ]; then + echo "$image" exists + else + echo "$image" missing! + exit 1 + fi +done From 155edf1a84c8ad89eae53f2daeb320988b66437c Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 12:46:27 -0800 Subject: [PATCH 04/13] test with built artifacts --- .github/workflows/tests.yml | 35 ++++++++++++++++++++++++++ src/main/c/h3-java/build-h3.sh | 2 +- src/main/c/h3-java/pull-from-github.sh | 2 +- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index adccd74..51acfad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -161,3 +161,38 @@ jobs: env: CI_NAME: github COVERALLS_SECRET: ${{ secrets.GITHUB_TOKEN }} + + tests-use-built-artifacts: + name: Java (Built Artifacts) ${{ matrix.java-version }} ${{ matrix.os }} + needs: + - tests + - tests-no-docker + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest] + java-distribution: [adopt] + java-version: [8] + + steps: + - uses: actions/checkout@v2.1.1 + with: + submodules: recursive + + - uses: actions/setup-java@v2 + with: + distribution: "${{ matrix.java-distribution }}" + java-version: "${{ matrix.java-version }}" + + - uses: actions/cache@v2 + id: maven-cache + with: + path: ~/.m2/ + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + + - name: Coverage report + run: | + mvn clean test -Dh3.github.artifacts=true diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 443617d..1688eb0 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -45,7 +45,7 @@ DOCKCROSS_TAG=$5 PULL_FROM_GITHUB=$6 if $PULL_FROM_GITHUB; then - ./pull-from-github.sh + src/main/c/h3-java//pull-from-github.sh exit 0 fi diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index 60496d0..1514632 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -42,7 +42,7 @@ echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do ARTIFACT_ID=$(echo $artifactline | jq .id) echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID" gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip" - unzip "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" + unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" done popd From 4a0fa19cafdfc228b00565c8cb7e958dff381987 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 13:09:01 -0800 Subject: [PATCH 05/13] add GH_TOKEN env for actions --- .github/workflows/tests.yml | 4 +++- src/main/c/h3-java/pull-from-github.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 51acfad..4fa8ecb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -193,6 +193,8 @@ jobs: restore-keys: | ${{ runner.os }}-maven- - - name: Coverage report + - name: Download and test run: | mvn clean test -Dh3.github.artifacts=true + env: + GH_TOKEN: ${{ github.token }} diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index 1514632..5c7ce18 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -22,7 +22,7 @@ # This script expects to be run from the project's base directory (where # pom.xml is) as part of the Maven build process. -set -ex +set -eox pipefail GIT_REVISION=$(git rev-parse HEAD) EXTRACT_TO=src/main/resources From a1455c89ab224884d61072031baf9011f8503128 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 15:33:08 -0800 Subject: [PATCH 06/13] provide github ref override --- .github/workflows/tests.yml | 2 +- docs/releasing.md | 4 ++-- pom.xml | 12 +++--------- src/main/c/h3-java/build-h3.sh | 18 +++++++++++++----- src/main/c/h3-java/pull-from-github.sh | 8 +++++++- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4fa8ecb..4944193 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -195,6 +195,6 @@ jobs: - name: Download and test run: | - mvn clean test -Dh3.github.artifacts=true + mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.ref=${{ github.ref }} env: GH_TOKEN: ${{ github.token }} diff --git a/docs/releasing.md b/docs/releasing.md index 04f1ac3..1231535 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -8,8 +8,8 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that 1. Ensure you are on branch `master` and that the library is building correctly. 2. Update [CHANGELOG.md](../CHANGELOG.md) to have the correct date and new version number, update [README.md](../README.md) to have the correct version numbers, and commit. -3. `mvn release:prepare -Dh3.github.artifacts=true` Use the new version number when prompted. -4. `mvn release:perform -Dh3.github.artifacts=true` +3. `mvn release:prepare -Dh3.github.artifacts.use=true` Use the new version number when prompted. +4. `mvn release:perform -Dh3.github.artifacts.use=true` 5. If this looks good, close and release the build in [Sonatype Nexus Manager](https://oss.sonatype.org/). 6. Update `CHANGELOG.md` to have an Unreleased section, and commit. The release is now done and development can resume from this point. diff --git a/pom.xml b/pom.xml index 63287a6..a7b052a 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,8 @@ true false 20210624-de7b1b0 - false + false + @@ -263,14 +264,7 @@ ${basedir}/src/main/c/h3-java/build-h3.sh - - ${h3.git.remote} - ${h3.git.reference} - ${h3.use.docker} - ${h3.system.prune} - ${h3.dockcross.tag} - ${h3.github.artifacts} - + ${h3.git.remote} ${h3.git.reference} ${h3.use.docker} ${h3.system.prune} ${h3.dockcross.tag} ${h3.github.artifacts.use} ${h3.github.artifacts.ref} diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 1688eb0..6eee7fa 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -15,7 +15,7 @@ # limitations under the License. # -# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] +# Arguments: [git-remote] [git-ref] [use-docker] [remove-images] [github-artifacts] [github-artifacts-ref] # git-remote - The git remote to pull from. An existing cloned repository # will not be deleted if a new remote is specified. # git-ref - Specific git ref of H3 to build. @@ -28,6 +28,8 @@ # github-artifacts - When set, all build artifacts are retrieved from Github # Actions artifacts rather than built locally (overrides # all other settings.) +# github-artifacts-ref - When set, override the default revision to pull artifacts from +# for `github-artifacts`. # # This script downloads H3, builds H3 and the H3-Java native library, and # cross compiles via Docker. @@ -42,11 +44,12 @@ GIT_REVISION=$2 USE_DOCKER=$3 SYSTEM_PRUNE=$4 DOCKCROSS_TAG=$5 -PULL_FROM_GITHUB=$6 +GITHUB_ARTIFACTS=$6 +GITHUB_ARTIFACTS_REF=$7 -if $PULL_FROM_GITHUB; then - src/main/c/h3-java//pull-from-github.sh - exit 0 +if $GITHUB_ARTIFACTS; then + src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_REF" + # The build of the library below is still needed for the binding-functions file fi echo Downloading H3 from "$GIT_REMOTE" @@ -99,6 +102,11 @@ cp h3-java-build/build/binding-functions . popd # target +if $GITHUB_ARTIFACTS; then + # Nothing more is needed than the binding-functions to stop now. + exit 0 +fi + # Copy the built artifact for this platform. case "$(uname -sm)" in "Linux x86_64") LIBRARY_DIR=linux-x64 ;; diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index 5c7ce18..4d07d9d 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -24,7 +24,13 @@ set -eox pipefail -GIT_REVISION=$(git rev-parse HEAD) +GITHUB_ARTIFACTS_REF=$1 + +if [ -z "$GITHUB_ARTIFACTS_REF" ]; then + GIT_REVISION=$(git rev-parse HEAD) +else + GIT_REVISION="$GITHUB_ARTIFACTS_REF" +fi EXTRACT_TO=src/main/resources echo downloading artifacts for $GIT_REVISION From 5d970ce33379f9e289f2fed0513c402db3221294 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 15:40:39 -0800 Subject: [PATCH 07/13] fix for windows --- pom.xml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index a7b052a..f2c6c30 100644 --- a/pom.xml +++ b/pom.xml @@ -125,14 +125,7 @@ powershell - - -ExecutionPolicy - Bypass - -File - ${basedir}/src/main/c/h3-java/build-h3-windows.ps1 - ${h3.git.remote} - ${h3.git.reference} - + -ExecutionPolicy Bypass -File ${basedir}/src/main/c/h3-java/build-h3-windows.ps1 ${h3.git.remote} ${h3.git.reference} From fa1bb06a36a8c8ed22c083d7a090c2323bcf3357 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 15:49:42 -0800 Subject: [PATCH 08/13] fix for windows --- pom.xml | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index f2c6c30..5b15b64 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,7 @@ org.codehaus.mojo exec-maven-plugin - 1.1.1 + 3.1.0 @@ -125,7 +125,14 @@ powershell - -ExecutionPolicy Bypass -File ${basedir}/src/main/c/h3-java/build-h3-windows.ps1 ${h3.git.remote} ${h3.git.reference} + + -ExecutionPolicy + Bypass + -File + ${project.basedir}/src/main/c/h3-java/build-h3-windows.ps1 + ${h3.git.remote} + ${h3.git.reference} + @@ -198,7 +205,7 @@ - ${basedir}/h3version.properties + ${project.basedir}/h3version.properties @@ -218,7 +225,7 @@ -h - ${basedir}/src/main/c/h3-java/src + ${project.basedir}/src/main/c/h3-java/src @@ -245,7 +252,7 @@ org.codehaus.mojo exec-maven-plugin - 1.1.1 + 3.1.0 build-h3-c @@ -256,8 +263,16 @@ exec - ${basedir}/src/main/c/h3-java/build-h3.sh - ${h3.git.remote} ${h3.git.reference} ${h3.use.docker} ${h3.system.prune} ${h3.dockcross.tag} ${h3.github.artifacts.use} ${h3.github.artifacts.ref} + ${project.basedir}/src/main/c/h3-java/build-h3.sh + + ${h3.git.remote} + ${h3.git.reference} + ${h3.use.docker} + ${h3.system.prune} + ${h3.dockcross.tag} + ${h3.github.artifacts.use} + ${h3.github.artifacts.ref} + @@ -277,7 +292,7 @@ ${project.build.outputDirectory} - ${basedir}/src/main/resources + ${project.basedir}/src/main/resources false From 78229d9d3e2a5c0ef61f73573bd64946d263684b Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 16:12:52 -0800 Subject: [PATCH 09/13] fix ci ref --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4944193..a53772b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -195,6 +195,6 @@ jobs: - name: Download and test run: | - mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.ref=${{ github.ref }} + mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.ref=${{ github.sha }} env: GH_TOKEN: ${{ github.token }} From 39e626a55761efc6c698a44e501faf7830156948 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 18:09:24 -0800 Subject: [PATCH 10/13] by run ID --- .github/workflows/tests.yml | 2 +- pom.xml | 4 ++-- src/main/c/h3-java/build-h3.sh | 6 +++--- src/main/c/h3-java/pull-from-github.sh | 25 ++++++++++++++----------- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a53772b..c5069ba 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -195,6 +195,6 @@ jobs: - name: Download and test run: | - mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.ref=${{ github.sha }} + mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=${{ github.run_id }} env: GH_TOKEN: ${{ github.token }} diff --git a/pom.xml b/pom.xml index 5b15b64..f8851db 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ false 20210624-de7b1b0 false - + @@ -271,7 +271,7 @@ ${h3.system.prune} ${h3.dockcross.tag} ${h3.github.artifacts.use} - ${h3.github.artifacts.ref} + ${h3.github.artifacts.by_run} diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 6eee7fa..660951c 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -28,7 +28,7 @@ # github-artifacts - When set, all build artifacts are retrieved from Github # Actions artifacts rather than built locally (overrides # all other settings.) -# github-artifacts-ref - When set, override the default revision to pull artifacts from +# github-artifacts-run - When set, override the default revision to pull artifacts from # for `github-artifacts`. # # This script downloads H3, builds H3 and the H3-Java native library, and @@ -45,10 +45,10 @@ USE_DOCKER=$3 SYSTEM_PRUNE=$4 DOCKCROSS_TAG=$5 GITHUB_ARTIFACTS=$6 -GITHUB_ARTIFACTS_REF=$7 +GITHUB_ARTIFACTS_RUN=$7 if $GITHUB_ARTIFACTS; then - src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_REF" + src/main/c/h3-java/pull-from-github.sh "$GITHUB_ARTIFACTS_RUN" # The build of the library below is still needed for the binding-functions file fi diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index 4d07d9d..d409e47 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -24,24 +24,27 @@ set -eox pipefail -GITHUB_ARTIFACTS_REF=$1 +GITHUB_ARTIFACTS_RUN=$1 -if [ -z "$GITHUB_ARTIFACTS_REF" ]; then - GIT_REVISION=$(git rev-parse HEAD) -else - GIT_REVISION="$GITHUB_ARTIFACTS_REF" -fi EXTRACT_TO=src/main/resources -echo downloading artifacts for $GIT_REVISION - mkdir -p target pushd target -TO_DOWNLOAD=$(gh api \ +ARTIFACTS_LIST=$(gh api \ -H "Accept: application/vnd.github+json" \ - /repos/{owner}/{repo}/actions/artifacts \ - | jq ".artifacts[] | select(.workflow_run.head_sha == \"$GIT_REVISION\")") + /repos/{owner}/{repo}/actions/artifacts) + +if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then + GIT_REVISION=$(git rev-parse HEAD) + echo "downloading artifacts for sha $GIT_REVISION" + TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \ + | jq ".artifacts[] | select(.workflow_run.head_sha == \"$GIT_REVISION\")") +else + echo "downloading artifacts for run $GITHUB_ARTIFACTS_RUN" + TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \ + | jq ".artifacts[] | select(.workflow_run.id == \"$GITHUB_ARTIFACTS_RUN\")") +fi echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do ARTIFACT_NAME=$(echo $artifactline | jq -r .name) From e35b5eb09f5449a3d6b08f7daf22c693400f46c4 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 19:14:50 -0800 Subject: [PATCH 11/13] Use GH actions to download --- .github/workflows/tests.yml | 18 ++++++++++--- src/main/c/h3-java/build-h3.sh | 4 +-- src/main/c/h3-java/pull-from-github.sh | 35 +++++++++++--------------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c5069ba..bc7139a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: matrix: os: [ubuntu-latest] java-distribution: [adopt] - java-version: [8, 11, 15, 17] + java-version: [8] #, 11, 15, 17] steps: - uses: actions/checkout@v2.1.1 @@ -96,7 +96,7 @@ jobs: # TODO: Docker on macos-latest running is not working os: [macos-latest, windows-latest] java-distribution: [adopt] - java-version: [8, 11, 15, 17] + java-version: [8] #, 11, 15, 17] steps: - uses: actions/checkout@v2.1.1 @@ -193,8 +193,20 @@ jobs: restore-keys: | ${{ runner.os }}-maven- + - name: Download Docker binaries + uses: actions/download-artifact@v3 + with: + name: docker-built-shared-objects + path: src/main/resources/ + + - name: Download Mac binaries + uses: actions/download-artifact@v3 + with: + name: macos-built-shared-objects + path: src/main/resources/ + - name: Download and test run: | - mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=${{ github.run_id }} + mvn clean test -Dh3.github.artifacts.use=true -Dh3.github.artifacts.by_run=true env: GH_TOKEN: ${{ github.token }} diff --git a/src/main/c/h3-java/build-h3.sh b/src/main/c/h3-java/build-h3.sh index 660951c..741a023 100755 --- a/src/main/c/h3-java/build-h3.sh +++ b/src/main/c/h3-java/build-h3.sh @@ -28,8 +28,8 @@ # github-artifacts - When set, all build artifacts are retrieved from Github # Actions artifacts rather than built locally (overrides # all other settings.) -# github-artifacts-run - When set, override the default revision to pull artifacts from -# for `github-artifacts`. +# github-artifacts-run - When set, assume the artifacts have already been +# downloaded. # # This script downloads H3, builds H3 and the H3-Java native library, and # cross compiles via Docker. diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index d409e47..47b7e82 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -28,33 +28,28 @@ GITHUB_ARTIFACTS_RUN=$1 EXTRACT_TO=src/main/resources -mkdir -p target -pushd target +if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then + mkdir -p target + pushd target -ARTIFACTS_LIST=$(gh api \ - -H "Accept: application/vnd.github+json" \ - /repos/{owner}/{repo}/actions/artifacts) + ARTIFACTS_LIST=$(gh api \ + -H "Accept: application/vnd.github+json" \ + /repos/{owner}/{repo}/actions/artifacts) -if [ -z "$GITHUB_ARTIFACTS_RUN" ]; then - GIT_REVISION=$(git rev-parse HEAD) - echo "downloading artifacts for sha $GIT_REVISION" - TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \ - | jq ".artifacts[] | select(.workflow_run.head_sha == \"$GIT_REVISION\")") -else echo "downloading artifacts for run $GITHUB_ARTIFACTS_RUN" TO_DOWNLOAD=$(echo "$ARTIFACTS_LIST" \ | jq ".artifacts[] | select(.workflow_run.id == \"$GITHUB_ARTIFACTS_RUN\")") -fi -echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do - ARTIFACT_NAME=$(echo $artifactline | jq -r .name) - ARTIFACT_ID=$(echo $artifactline | jq .id) - echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID" - gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip" - unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" -done + echo $TO_DOWNLOAD | jq -c '.' | while read artifactline; do + ARTIFACT_NAME=$(echo $artifactline | jq -r .name) + ARTIFACT_ID=$(echo $artifactline | jq .id) + echo "Downloading $ARTIFACT_NAME: $ARTIFACT_ID" + gh api "/repos/{owner}/{repo}/actions/artifacts/$ARTIFACT_ID/zip" > "$ARTIFACT_NAME.zip" + unzip -o "$ARTIFACT_NAME.zip" -d "../$EXTRACT_TO" + done -popd + popd +fi echo Checking that expected images are present: From fc65a0fec8a116b54dd44ead9a85a982e5c6c3a7 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Sun, 29 Jan 2023 19:42:17 -0800 Subject: [PATCH 12/13] reenable jobs --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc7139a..d36f68d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: matrix: os: [ubuntu-latest] java-distribution: [adopt] - java-version: [8] #, 11, 15, 17] + java-version: [8, 11, 15, 17] steps: - uses: actions/checkout@v2.1.1 @@ -96,7 +96,7 @@ jobs: # TODO: Docker on macos-latest running is not working os: [macos-latest, windows-latest] java-distribution: [adopt] - java-version: [8] #, 11, 15, 17] + java-version: [8, 11, 15, 17] steps: - uses: actions/checkout@v2.1.1 From b6c4f642cedb47df55d01809b5e25ee6e2f7c888 Mon Sep 17 00:00:00 2001 From: Isaac Brodsky Date: Mon, 30 Jan 2023 14:20:51 -0800 Subject: [PATCH 13/13] comment updates --- docs/releasing.md | 5 +++-- src/main/c/h3-java/pull-from-github.sh | 12 +++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index 1231535..c55e587 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -15,9 +15,10 @@ Release builds should only be run on Mac OSX with Docker. This is needed so that ## Troubleshooting -### Github artifacts +### Dependencies for `pull-from-github.sh` -You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope. +* You should install the [Github CLI](https://cli.github.com) and authenticate with it first. You may need to use a personal access token (classic) with workflows scope. +* `jq` ### gpg: signing failed: Inappropriate ioctl for device diff --git a/src/main/c/h3-java/pull-from-github.sh b/src/main/c/h3-java/pull-from-github.sh index 47b7e82..4b26ce0 100755 --- a/src/main/c/h3-java/pull-from-github.sh +++ b/src/main/c/h3-java/pull-from-github.sh @@ -15,9 +15,15 @@ # limitations under the License. # -# Retrieves built artifacts from Github. You must install the Github CLI -# and authenticate with a personal access token (classic) with workflows -# scope to use this. https://cli.github.com/ +# Retrieves built artifacts from Github. The purpose of this script is +# to enable deployment from any system, regardless of whether it can +# run the particular cross compiling steps needed to build native shared +# objects. Some of the native shared objects require particular Docker +# setups, or indeed running on Mac OS, which is difficult to maintain. +# +# You must install the Github CLI and authenticate with a personal access +# token (classic) with workflows scope to use this. https://cli.github.com/ +# This script also requires `jq`. # # This script expects to be run from the project's base directory (where # pom.xml is) as part of the Maven build process.