Merge branch 'main' into bump-apm-perf

Author: carsonip

.buildkite/hooks/pre-command

@@ -10,25 +10,8 @@
 
 set -eo pipefail
 
-retry() {
-    local retries=$1
-    shift
-
-    local count=0
-    until "$@"; do
-        exit=$?
-        wait=$((2 ** count))
-        count=$((count + 1))
-        if [ $count -lt "$retries" ]; then
-            >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
-            sleep $wait
-        else
-            >&2 echo "Retry $count/$retries exited $exit, no more retries left."
-            return $exit
-        fi
-    done
-    return 0
-}
+# shellcheck disable=SC1091
+source .buildkite/scripts/utils.sh
 
 get_os_details() {
     case $(uname | tr '[:upper:]' '[:lower:]') in
@@ -93,7 +76,7 @@ GH_TOKEN="${VAULT_GITHUB_TOKEN}"
 export GH_TOKEN
 
 echo "--- Setting up the :golang: environment..."
-GO_VERSION=$(cat .go-version)
+GO_VERSION=$(grep '^go' go.mod | cut -d' ' -f2)
 WORKSPACE=$(git rev-parse --show-toplevel)
 export GO_VERSION WORKSPACE
 get_os_details

.buildkite/scripts/dra.sh

@@ -13,8 +13,9 @@ set -eo pipefail
 # Either staging or snapshot
 TYPE="$1"
 
-## Read current version without the qualifier
-VERSION=$(make get-version-only)
+# NOTE: load the shared functions
+# shellcheck disable=SC1091
+source .buildkite/scripts/utils.sh
 
 echo "--- Restoring Artifacts"
 buildkite-agent artifact download "build/**/*" .
@@ -34,48 +35,26 @@ fi
 # by default it uses the buildkite branch
 DRA_BRANCH="$BUILDKITE_BRANCH"
 # by default it publishes the DRA artifacts, for such it uses the collect command.
-dra_command=collect
+DRA_COMMAND=collect
+VERSION=$(make get-version-only)
 BRANCHES_URL=https://storage.googleapis.com/artifacts-api/snapshots/branches.json
 curl -s "${BRANCHES_URL}" > active-branches.json
-# as long as `8.x` is not in the active branches, we will explicitly add the condition.
-if [ "$BUILDKITE_BRANCH" == "8.x" ] || grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
-  echo "--- :arrow_right: Release Manager only supports the current active branches and 8.x, running"
-else
+if ! grep -q "\"$BUILDKITE_BRANCH\"" active-branches.json ; then
   # If no active branches are found, let's see if it is a feature branch.
-  echo "--- :arrow_right: Release Manager only supports the current active branches, skipping"
-  echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
-  echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
-  echo "VERSION=$VERSION"
-  echo "Supported branches:"
-  cat active-branches.json
-  if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
-    buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
-    dra_command=list
-
-    # use a different branch since DRA does not support feature branches but main/release branches
-    # for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
-    # to know if the branch was branched out from main or the release branches.
-    MAJOR_MINOR=${VERSION%.*}
-    if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
-      DRA_BRANCH=main
-    else
-      if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
-        DRA_BRANCH="$MAJOR_MINOR"
-      else
-        buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
-        exit 0
-      fi
-    fi
-  else
-    buildkite-agent annotate "${BUILDKITE_BRANCH} is not supported yet. Look for the supported branches in ${BRANCHES_URL}" --style 'warning' --context 'ctx-warn'
-    exit 1
-  fi
+  dra_process_other_branches
 fi
 
+echo "--- :arrow_right: Release Manager only supports the current active branches"
+echo "BUILDKITE_BRANCH=$BUILDKITE_BRANCH"
+echo "BUILDKITE_COMMIT=$BUILDKITE_COMMIT"
+echo "VERSION=$VERSION"
+echo "Supported branches:"
+cat active-branches.json
+
 dra() {
   local workflow=$1
   local command=$2
-  local qualifier=${ELASTIC_QUALIFIER:-""}
+  local qualifier=${3:-""}
   echo "--- Run release manager $workflow (DRA command: $command)"
   set -x
   docker run --rm \
@@ -94,32 +73,20 @@ dra() {
       --qualifier "$qualifier" \
       --version $VERSION | tee rm-output.txt
   set +x
-  # Create Buildkite annotation similarly done in Beats:
-  # https://github.com/elastic/beats/blob/90f9e8f6e48e76a83331f64f6c8c633ae6b31661/.buildkite/scripts/dra.sh#L74-L81
-  if [[ "$command" == "collect" ]]; then
-    # extract the summary URL from a release manager output line like:
-    # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/apm-server/18.22.0-ABCDEFGH/summary-18.22.0.html
-    SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}')
-    rm rm-output.txt
 
-    # and make it easily clickable as a Builkite annotation
-    printf "**${workflow} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
-  fi
+  create_annotation_dra_summary "$command" "$workflow" rm-output.txt
 }
 
 if [[ "${TYPE}" == "staging" ]]; then
+  qualifier=$(fetch_elastic_qualifier "$DRA_BRANCH")
+  # TODO: main and 8.x are not needed to run the DRA for staging
+  #       but main is needed until we do alpha1 releases of 9.0.0
   if [[ "${DRA_BRANCH}" != "8.x" ]]; then
-    echo "${DRA_BRANCH} is not '8.x'"
-    if [[ "${DRA_BRANCH}" == "main" ]] ; then
-      # NOTE: qualifier is needed for main/staging at the moment. Skip builds if no ELASTIC_QUALIFIER
-      if [[ -n "${ELASTIC_QUALIFIER}" ]]; then
-        dra "${TYPE}" "$dra_command"
-      fi
-    else
-      dra "${TYPE}" "$dra_command"
-    fi
+    dra "${TYPE}" "$DRA_COMMAND" "${qualifier}"
   fi
-else
+fi
+
+if [[ "${TYPE}" == "snapshot" ]]; then
   # NOTE: qualifier is not needed for snapshots, let's unset it.
-  dra "snapshot" "$dra_command" ""
+  dra "${TYPE}" "$DRA_COMMAND" ""
 fi

.buildkite/scripts/package.sh

@@ -19,6 +19,17 @@ if [[ ${TYPE} == "snapshot" ]]; then
   MAKE_GOAL="${MAKE_GOAL}-snapshot"
 fi
 
+if [[ ${TYPE} == "staging" ]]; then
+  echo "--- Prepare the Elastic Qualifier"
+  # NOTE: load the shared functions
+  # shellcheck disable=SC1091
+  source .buildkite/scripts/utils.sh
+  dra_process_other_branches
+  ELASTIC_QUALIFIER=$(fetch_elastic_qualifier "$DRA_BRANCH")
+  export ELASTIC_QUALIFIER
+fi
+
+echo "--- Run $MAKE_GOAL for $DRA_BRANCH"
 make $MAKE_GOAL
 
 ls -l build/distributions/

.buildkite/scripts/utils.sh

@@ -0,0 +1,80 @@
+# An opinionated approach to manage the Elatic Qualifier for the DRA in a Google Bucket
+# Instead of using the ELASTIC_QUALIFIER env variable.
+fetch_elastic_qualifier() {
+  local branch=$1
+  qualifier=""
+  URL="https://storage.googleapis.com/dra-qualifier/$branch"
+  if curl -sf -o /dev/null "$URL" ; then
+    qualifier=$(curl -s "$URL")
+  fi
+  echo "$qualifier"
+}
+
+retry() {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** count))
+    count=$((count + 1))
+    if [ $count -lt "$retries" ]; then
+      >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      >&2 echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+  done
+  return 0
+}
+
+#
+# An opinionated approach to detect if unsupported Unified Release branches
+# can be used, this is handy for testing feature branches in dry-run mode
+# It produces the below environment variables:
+# - VERSION
+# - DRA_COMMAND
+# - DRA_BRANCH
+dra_process_other_branches() {
+  ## Read current version without the qualifier
+  VERSION=$(make get-version-only)
+  DRA_BRANCH="$BUILDKITE_BRANCH"
+  if [[ $BUILDKITE_BRANCH =~ "feature/" ]]; then
+    buildkite-agent annotate "${BUILDKITE_BRANCH} will list DRA artifacts. Feature branches are not supported. Look for the supported branches in ${BRANCHES_URL}" --style 'info' --context 'ctx-info'
+    DRA_COMMAND=list
+
+    # use a different branch since DRA does not support feature branches but main/release branches
+    # for such we will use the VERSION and https://storage.googleapis.com/artifacts-api/snapshots/<major.minor>.json
+    # to know if the branch was branched out from main or the release branches.
+    MAJOR_MINOR=${VERSION%.*}
+    if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/main.json" | grep -q "$VERSION" ; then
+      DRA_BRANCH=main
+    else
+      if curl -s "https://storage.googleapis.com/artifacts-api/snapshots/$MAJOR_MINOR.json" | grep -q "$VERSION" ; then
+        DRA_BRANCH="$MAJOR_MINOR"
+      else
+        buildkite-agent annotate "It was not possible to know the original base branch for ${BUILDKITE_BRANCH}. This won't fail - this is a feature branch." --style 'info' --context 'ctx-info-feature-branch'
+      fi
+    fi
+  fi
+  export DRA_BRANCH DRA_COMMAND VERSION
+}
+
+# Create Buildkite annotation similarly done in Beats:
+# https://github.com/elastic/beats/blob/90f9e8f6e48e76a83331f64f6c8c633ae6b31661/.buildkite/scripts/dra.sh#L74-L81
+create_annotation_dra_summary() {
+  local command=$1
+  local workflow=$2
+  local output=$3
+  if [[ "$command" == "collect" ]]; then
+    # extract the summary URL from a release manager output line like:
+    # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/apm-server/18.22.0-ABCDEFGH/summary-18.22.0.html
+    SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' "$output" | grep -oP 'https://\S+' | awk '{print $1}')
+    rm "$output"
+
+    # and make it easily clickable as a Builkite annotation
+    printf "**${workflow} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append
+  fi
+}

.ci/scripts/bench.sh

@@ -170,7 +170,7 @@ if [ "${CI}" == "true" ] ; then
   export BUILDKITE_TOKEN="${BUILDKITE_TOKEN_SECRET}"
 
   echo "--- Setting up the :golang: environment..."
-  GO_VERSION=$(cat .go-version)
+  GO_VERSION=$(grep '^go' go.mod | cut -d' ' -f2)
   OS_NAME=linux
   OS_ARCH=amd64
   retry 5 curl -sL -o "${BASE_PROJECT}"/gvm "https://github.com/andrewkroh/gvm/releases/download/v0.5.2/gvm-${OS_NAME}-${OS_ARCH}"

.ci/scripts/unit-test.sh

@@ -35,9 +35,14 @@ on:
         description: 'Set the expression that matches the benchmark scenarios to run'
         required: false
         type: string
+      pgoExport:
+        description: 'Run the benchmarks with PGO export'
+        required: false
+        type: boolean
+        default: false
   schedule:
     - cron: '0 17 * * *' # Scheduled regular benchmarks.
-    - cron: '0 5 */5 * *' # Scheduled PGO benchmarks.
+    - cron: '0 0 1 * *' # Scheduled PGO benchmarks.
 
 env:
   PNG_REPORT_FILE: out.png
@@ -61,10 +66,11 @@ jobs:
       SSH_KEY: ./id_rsa_terraform
       TF_VAR_private_key: ./id_rsa_terraform
       TF_VAR_public_key: ./id_rsa_terraform.pub
-      TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
+      TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
       TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work
       TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work
-      RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 5 */5 * *' }}
+      RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
+      PGO_EXPORT: ${{ inputs.pgoExport || github.event.schedule=='0 0 1 * *' }}
       TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile
       TF_VAR_BUILD_ID: ${{ github.run_id }}
       TF_VAR_ENVIRONMENT: ci
@@ -97,6 +103,11 @@ jobs:
             echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV"
           fi
 
+      # exclude BenchmarkTraces* from scheduled benchmarks
+      - if: github.event_name == 'schedule'
+        run: |
+          echo "BENCHMARK_RUN=Benchmark[^T]" >> "$GITHUB_ENV"
+
       - name: Log in to the Elastic Container registry
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
@@ -110,7 +121,7 @@ jobs:
         with:
           role-duration-seconds: 21600 # 6 hours
 
-      - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
+      - uses: google-github-actions/get-secretmanager-secrets@a8440875e1c2892062aef9061228d4f1af8f919b # v2.2.3
         with:
           export_to_environment: true
           secrets: |-
@@ -228,13 +239,13 @@ jobs:
       - name: Import GPG key
         uses: crazy-max/ghaction-import-gpg@cb9bde2e2525e640591a934b1fd28eef1dcaf5e5  # v6.2.0
         with:
-          gpg_private_key: ${{ secrets.APM_SERVER_RELEASE_GPG_PRIVATE_KEY }}
-          passphrase: ${{ secrets.APM_SERVER_RELEASE_PASSPHRASE }}
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.GPG_PASSPHRASE }}
           git_user_signingkey: true
           git_commit_gpgsign: true
 
       - name: Open PGO PR
-        if: ${{ env.RUN_STANDALONE == 'true' }}
+        if: ${{ env.PGO_EXPORT == 'true' }}
         run: ${{ github.workspace }}/.ci/scripts/push-pgo-pr.sh
         env:
           WORKSPACE_PATH: ${{ github.workspace }}
@@ -246,7 +257,7 @@ jobs:
       # there is a high chance things will stop working
       # This is trying to reduce the chances of that happening.
       # See https://github.com/elastic/observability-test-environments/actions/workflows/cluster-rotate-api-keys.yml
-      - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
+      - uses: google-github-actions/get-secretmanager-secrets@a8440875e1c2892062aef9061228d4f1af8f919b # v2.2.3
         if: always()
         with:
           export_to_environment: true

.github/workflows/benchmarks.yml

@@ -6,6 +6,7 @@ on:
       - main
       - 7.1*
       - 8.*
+      - 9.*
     paths-ignore: # When updating the list of expressions also update ci-docs.yml
       - '**.md'
       - '**.asciidoc'
@@ -36,9 +37,6 @@ jobs:
             go.sum
             tools/go.sum
       - run: make check-full
-      - name: Ensure .go-version is up to date
-        run: make update-go-version
-        if: ${{ github.ref == 'main' }}
 
   test:
     strategy:

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
 
       - uses: elastic/oblt-actions/google/auth@v1
 
-      - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2.2.2
+      - uses: google-github-actions/get-secretmanager-secrets@a8440875e1c2892062aef9061228d4f1af8f919b # v2.2.3
         with:
           export_to_environment: true
           secrets: |-

.github/workflows/functional-tests.yml

@@ -7,8 +7,8 @@ on:
   push:
     branches:
       - main
-      - "8.[0-9]+"
-      - "8.x"
+      - "8.*"
+      - "9.*"
     paths-ignore:
       - '**.md'
       - '**.asciidoc'