From f78154d318810c9f2cb0249d19dd33b8d4fa722a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 22 Oct 2021 14:06:25 +0200 Subject: [PATCH 1/3] solc-bin: Scripts for downloading and renaming binaries from CircleCI job artifacts --- .../solc-bin/download-circleci-binaries.sh | 207 ++++++++++++++++++ .../rename-circleci-binaries-for-solc-bin.sh | 52 +++++ 2 files changed, 259 insertions(+) create mode 100755 scripts/solc-bin/download-circleci-binaries.sh create mode 100755 scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh diff --git a/scripts/solc-bin/download-circleci-binaries.sh b/scripts/solc-bin/download-circleci-binaries.sh new file mode 100755 index 000000000000..0f70f55e369a --- /dev/null +++ b/scripts/solc-bin/download-circleci-binaries.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash + +set -euo pipefail + +function help() { + echo "Downloads binaries created by a CircleCI job to the current working," + echo "directory and renames them according to the naming convention used" + echo "on the Solidity release page on Github." + echo + echo "WARNING: The binaries will be overwritten if they already exist." + echo + echo + echo "Usage:" + echo " ./$(basename "$0") --help" + echo " ./$(basename "$0") workflow_id" + echo + echo " workflow_id CircleCI workflow ID. Identifies the workflow that contains" + echo " jobs whose artifacts should be downloaded." + echo + echo + echo "Examples:" + echo " ./$(basename "$0") --help" + echo " ./$(basename "$0") 12345678-90ab-cdef-1234-567890abcdef" +} + + +# GENERAL UTILITIES + +query_api() { + local api_endpoint="$1" + + curl --fail --silent --show-error "$api_endpoint" +} + +fail() { + local format="$1" + + # shellcheck disable=SC2059 + >&2 printf "ERROR: $format\n" "${@:2}" + exit 1 +} + +expect_single_line() { + local text="$1" + + local line_count; line_count="$(echo "$text" | grep --count "")" + [[ $text != "" ]] || fail "Expected one line, got zero." + (( "$line_count" < 2 )) || fail "Expected one line, got %d:\n%s" "$line_count" "$text" +} + + +# JSON FILTERING + +filter_artifacts_by_name() { + local artifact_name="$1" + + jq '[ .items[] | select (.path == "'"$artifact_name"'") ]' +} + +filter_jobs_by_name() { + local job_name="$1" + + jq '[ .items[] | select (.name == "'"$job_name"'") ]' +} + + +# ENUMERATIONS + +target_to_job_name() { + local target="$1" + + case "$target" in + linux-amd64) echo b_ubu_static ;; + macosx-amd64) echo b_osx ;; + windows-amd64) echo b_win_release ;; + emscripten) echo b_ems ;; + *) fail "Invalid target: %s" "$target" ;; + esac +} + +target_to_circleci_artifact_name() { + local target="$1" + + case "$target" in + linux-amd64) echo solc ;; + macosx-amd64) echo solc ;; + windows-amd64) echo upload/bin/solc.exe ;; + emscripten) echo soljson.js ;; + *) fail "Invalid target: %s" "$target" ;; + esac +} + +is_executable() { + local target="$1" + + case "$target" in + linux-amd64) true ;; + macosx-amd64) true ;; + windows-amd64) false ;; + emscripten) false ;; + *) fail "Invalid target: %s" "$target" ;; + esac +} + +target_to_binary_name() { + local target="$1" + + case "$target" in + linux-amd64) echo solc-static-linux ;; + macosx-amd64) echo solc-macos ;; + windows-amd64) echo solc-windows.exe ;; + emscripten) echo soljson.js ;; + *) fail "Invalid target: %s" "$target" ;; + esac +} + + +# MAIN LOGIC + +query_circleci_artifact_url() { + local target="$1" + local build_num="$2" + + local artifact_name; artifact_name="$(target_to_circleci_artifact_name "$target")" + local artifact_info; artifact_info="$( + query_api "https://circleci.com/api/v2/project/gh/ethereum/solidity/${build_num}/artifacts" | + filter_artifacts_by_name "$artifact_name" + )" + + [[ $artifact_info != "[]" ]] || fail "Artifact '%s' is missing." "$artifact_name" + + local artifact_url; artifact_url="$(echo "$artifact_info" | jq --raw-output '.[].url')" + expect_single_line "$artifact_url" + + echo "$artifact_url" +} + +download_binary() { + local target="$1" + local download_url="$2" + + local target_path; target_path="$(target_to_binary_name "$target")" + + # If the target exists we ovewrite it. As a special case, if it's a symlink, remove it + # so that we only change link not the file it links to. + [[ ! -L "$target_path" ]] || rm "$target_path" + + echo "Downloading release binary from ${download_url} into ${target_path}" + curl "$download_url" --output "$target_path" --location --no-progress-meter --create-dirs + + ! is_executable "$target" || chmod +x "$target_path" +} + +download_binary_from_circleci() { + local target="$1" + local workflow_job_info="$2" + + local job_name; job_name="$(target_to_job_name "$target")" + local job_info; job_info="$( + echo "$workflow_job_info" | + filter_jobs_by_name "$job_name" + )" + [[ $job_info != "[]" ]] || fail "Job '%s' not found." "$job_name" + + local build_num; build_num="$(echo "$job_info" | jq --raw-output '.[].job_number')" + expect_single_line "$build_num" + + local artifact_url; artifact_url="$(query_circleci_artifact_url "$target" "$build_num")" + download_binary "$target" "$artifact_url" +} + +download_binaries() { + local workflow_id="$1" + + echo "===> DOWNLOADING BINARIES FOR WORKFLOW ${workflow_id} FROM CIRCLECI" + + local workflow_job_info; workflow_job_info="$( + query_api "https://circleci.com/api/v2/workflow/${workflow_id}/job" + )" + + local release_targets=( + linux-amd64 + windows-amd64 + macosx-amd64 + emscripten + ) + + for target in "${release_targets[@]}"; do + download_binary_from_circleci "$target" "$workflow_job_info" + done +} + +main() { + if (( $# > 0 )) && [[ $1 == --help ]]; then + help + exit 0 + fi + + (( $# >= 1 )) || fail "Too few arguments" + (( $# <= 1 )) || fail "Too many arguments" + + local workflow_id="${1:-$PWD}" + + download_binaries "$workflow_id" +} + +main "$@" diff --git a/scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh b/scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh new file mode 100755 index 000000000000..5706888dcd95 --- /dev/null +++ b/scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if (( $# > 0 )) && [[ $1 == --help ]]; then + echo "Renames binaries following the naming convention from Solidity release page on Github to match" + echo "the naming convention used in solc-bin." + echo "Assumes that the binaries are present in the current working directory." + echo "Obtains version from the --version output of the Linux binary." + echo + echo "WARNING: The binaries will be overwritten if they already exist." + echo + echo + echo "Usage:" + echo " ./$(basename "$0") --help" + echo " ./$(basename "$0") [solc_bin_dir]" + echo + echo " solc_bin_dir Location of the solc-bin directory." + echo " Default: current working directory." + echo + echo + echo "Examples:" + echo " ./$(basename "$0") --help" + echo " ./$(basename "$0") ~/solc-bin/" + + exit 0 +fi + +(( $# <= 1 )) || { >&2 printf "ERROR: Too many arguments"; exit 1; } + +solc_bin_dir="${1:-$PWD}" + +full_version=$( + ./solc-static-linux --version | + sed -En 's/^Version: ([0-9.]+.*\+commit\.[0-9a-f]+(\.mod)?).*$/\1/p' +) + +target=linux-amd64 +mkdir -p "${solc_bin_dir}/${target}" +mv solc-static-linux "${solc_bin_dir}/${target}/solc-${target}-${full_version}" + +target=macosx-amd64 +mkdir -p "${solc_bin_dir}/${target}" +mv solc-macos "${solc_bin_dir}/${target}/solc-${target}-${full_version}" + +target=windows-amd64 +mkdir -p "${solc_bin_dir}/${target}" +mv solc-windows.exe "${solc_bin_dir}/${target}/solc-${target}-${full_version}.exe" + +target=bin +mkdir -p "${solc_bin_dir}/${target}" +mv soljson.js "${solc_bin_dir}/${target}/soljson-${full_version}.js" From 402668a7eb2d7005c0f0a9bf4003dafdf15db15f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 22 Oct 2021 14:07:19 +0200 Subject: [PATCH 2/3] CI: Add b_release_binaries job that gathers release binaries from individual jobs --- .circleci/config.yml | 40 ++++++++++++++++++++++++++++++++++++++++ ReleaseChecklist.md | 23 +++++------------------ 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a75e372ae32f..f26f880ad2dd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1401,6 +1401,32 @@ jobs: path: all-bytecode-reports.zip - gitter_notify_failure_unless_pr + b_release_binaries: + <<: *base_ubuntu2004 + steps: + - checkout + - run: + name: Install dependencies + command: | + apt-get update + apt-get install --assume-yes --no-install-recommends jq + - run: + name: Download artifacts of dependent jobs + command: | + mkdir github/ + cd github/ + ../scripts/solc-bin/download-circleci-binaries.sh "$CIRCLE_WORKFLOW_ID" + - store_artifacts: + path: github/ + - run: + name: Rename binaries to solc-bin naming convention + command: | + mv github/ solc-bin/ + cd solc-bin/ + ../scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh + - store_artifacts: + path: solc-bin/ + workflows: version: 2 @@ -1519,6 +1545,20 @@ workflows: - b_bytecode_osx - b_bytecode_ems + # Final artifacts + - b_release_binaries: + filters: + tags: + # We only need release binaries from tagged commits + only: /^v.*/ + branches: + ignore: /.*/ + requires: + - b_ubu_static + - b_osx + - b_win_release + - b_ems + nightly: triggers: diff --git a/ReleaseChecklist.md b/ReleaseChecklist.md index 6d5008cb37fb..144343c931b9 100644 --- a/ReleaseChecklist.md +++ b/ReleaseChecklist.md @@ -26,29 +26,16 @@ - [ ] Click the `PUBLISH RELEASE` button on the release page, creating the tag. - [ ] Wait for the CI runs on the tag itself. -### Upload Release Artifacts +### Upload Release Artifacts and Publish Binaries - [ ] Switch to the tag that archives have to be created for. - [ ] Create the ``prerelease.txt`` file: (``echo -n > prerelease.txt``). - [ ] Run ``scripts/create_source_tarball.sh`` while being on the tag to create the source tarball. This will create the tarball in a directory called ``upload``. - [ ] Take the tarball from the upload directory (its name should be ``solidity_x.x.x.tar.gz``, otherwise ``prerelease.txt`` was missing in the step before) and upload the source tarball to the release page. - - [ ] Take the ``solc.exe`` binary from the ``b_win_release`` run of the released commit in circle-ci and add it to the release page as ``solc-windows.exe``. - - [ ] Take the ``solc`` binary from the ``b_osx`` run of the released commit in circle-ci and add it to the release page as ``solc-macos``. - - [ ] Take the ``solc`` binary from the ``b_ubu_static`` run of the released commit in circle-ci and add it to the release page as ``solc-static-linux``. - - [ ] Take the ``soljson.js`` binary from the ``b_ems`` run of the released commit in circle-ci and add it to the release page as ``soljson.js``. - -### Update [solc-bin](https://github.com/ethereum/solc-bin/) - - [ ] Copy files to solc-bin: - ```bash - VERSION=0.8.4 - COMMIT="c7e474f2" - SOLC_BIN="/home/me/solc-bin" - chmod +x solc-static-linux solc-macos - cp soljson.js $SOLC_BIN/bin/soljson-v$VERSION+commit.$COMMIT.js - cp solc-static-linux $SOLC_BIN/linux-amd64/solc-linux-amd64-v$VERSION+commit.$COMMIT - cp solc-macos $SOLC_BIN/macosx-amd64/solc-macosx-amd64-v$VERSION+commit.$COMMIT - cp solc-windows.exe $SOLC_BIN/windows-amd64/solc-windows-amd64-v$VERSION+commit.$COMMIT.exe + - [ ] Take the ``github/`` directory from ``b_release_binaries`` run of the tagged commit in circle-ci and add all binaries from it to the release page. + Make sure it contains four binaries: ``solc-windows.exe``, ``solc-macos``, ``solc-static-linux`` and ``soljson.js``. + - [ ] Take the ``solc-bin/`` directory from ``b_release_binaries`` run of the tagged commit in circle-ci and add all binaries from it to solc-bin. - [ ] Run ``./update --reuse-hashes`` in ``solc-bin`` and verify that the script has updated ``list.js``, ``list.txt`` and ``list.json`` files correctly and that symlinks to the new release have been added in ``solc-bin/wasm/`` and ``solc-bin/emscripten-wasm32/``. - - [ ] Create a pull request and merge. + - [ ] Create a pull request in solc-bin and merge. ### Homebrew and MacOS - [ ] Update the version and the hash (``sha256sum solidity_$VERSION.tar.gz``) in https://github.com/Homebrew/homebrew-core/blob/master/Formula/solidity.rb From e32daefb286dff71bc6e6625502b9c3dc079f0eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 22 Oct 2021 14:44:21 +0200 Subject: [PATCH 3/3] CI: Post a notification to gitter when binaries are ready --- .circleci/config.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f26f880ad2dd..a076018a63d8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ commands: parameters: event: type: enum - enum: ["failure", "success"] + enum: ["failure", "success", "release"] condition: type: string steps: @@ -58,6 +58,10 @@ commands: [[ "<< parameters.event >>" == "failure" ]] && message=" ❌ [${workflow_name}] Job ${job} failed on **${CIRCLE_BRANCH}**. Please see [build ${CIRCLE_BUILD_NUM}](${CIRCLE_BUILD_URL}) for details." [[ "<< parameters.event >>" == "success" ]] && message=" ✅ [${workflow_name}] Job ${job} succeeded on **${CIRCLE_BRANCH}**. Please see [build ${CIRCLE_BUILD_NUM}](${CIRCLE_BUILD_URL}) for details." + [[ "<< parameters.event >>" == "release" ]] && message=" 📦 Release binaries for version **${CIRCLE_TAG}** are ready and attached as artifacts to [build ${CIRCLE_BUILD_NUM}](${CIRCLE_BUILD_URL}). **Please make sure the whole workflow succeeded before using them.**" + + # The release notification only makes sense on tagged commits + [[ "<< parameters.event >>" == "release" ]] && { [[ $CIRCLE_TAG != "" ]] || { echo "Not a tagged commit - notification skipped."; exit 0; } } curl "https://api.gitter.im/v1/rooms/${GITTER_NOTIFY_ROOM_ID}/chatMessages" \ --request POST \ @@ -81,6 +85,13 @@ commands: event: success condition: on_success + gitter_notify_release_unless_pr: + description: "Posts a release notification to the main room on Gitter (if not running on a PR)." + steps: + - gitter_notify_unless_pr: + event: release + condition: on_success + defaults: # -------------------------------------------------------------------------- @@ -1426,6 +1437,7 @@ jobs: ../scripts/solc-bin/rename-circleci-binaries-for-solc-bin.sh - store_artifacts: path: solc-bin/ + - gitter_notify_release_unless_pr workflows: version: 2