LLVM Apple & FPM pipelines migration (#330)

elastic · Nov 27, 2023 · e2878ba · e2878ba
1 parent d3e2bdb
commit e2878ba
Show file tree

Hide file tree

Showing 13 changed files with 164 additions and 25 deletions.
diff --git a/.buildkite/fpm-pipeline.yml b/.buildkite/fpm-pipeline.yml
@@ -1,3 +1,27 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  DOCKER_REGISTRY: "docker.elastic.co"
+  STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
+  MAKEFILE: "fpm"
+  CHANGESET_FILE: ".buildkite/scripts/changeset/fpm"
+  DOCKER_FILTER_REF: "docker.elastic.co/beats-dev"
+  BUILDX: "0"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: ":linux: Build FPM / Ubuntu X86_64"
+    key: "build-ubuntu-x86"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
+    command:
+      - ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
+      - ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+    notify:
+      - github_commit_status:
+          context: "Build FPM / Ubuntu X86_64"
+    env:
+      REPOSITORY: "${STAGING_IMAGE}"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -11,11 +11,11 @@ GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" =~ ^build ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == build* ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ("$BUILDKITE_STEP_KEY" =~ ^build || "$BUILDKITE_STEP_KEY" =~ ^publish)]]; then
+if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple"  || "$BUILDKITE_PIPELINE_SLUG" == "fpm") && ( "$BUILDKITE_STEP_KEY" == build* ) ]]; then
     export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
     export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null

diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
@@ -6,11 +6,11 @@ source .buildkite/scripts/common.sh
 
 unset_secrets
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" =~ ^build ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == build* ]]; then
     google_cloud_logout_active_account
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ("$BUILDKITE_STEP_KEY" =~ ^build || "$BUILDKITE_STEP_KEY" =~ ^publish) ]]; then
+if [[ ( "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple" || "$BUILDKITE_PIPELINE_SLUG" == "fpm") && ( "$BUILDKITE_STEP_KEY" == build* ) ]]; then
     docker logout "${DOCKER_REGISTRY}"
 fi
 

diff --git a/.buildkite/llvm-apple-pipeline.yml b/.buildkite/llvm-apple-pipeline.yml
@@ -1,3 +1,59 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  SETUP_GVM_VERSION: "v0.5.1"
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  DOCKER_REGISTRY: "docker.elastic.co"
+  STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
+  MAKEFILE: "go/llvm-apple"
+  CHANGESET_FILE: ".buildkite/scripts/changeset/llvm-apple"
+  DOCKER_FILTER_REF: "*/*/golang-crossbuild:llvm-apple*"
+  BUILDX: "0"
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
+    key: "build-ubuntu-x86"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
+    command:
+      - ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
+      - ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+    notify:
+      - github_commit_status:
+          context: "Build LLVM Apple / Ubuntu X86_64"
+    env:
+      TAG_EXTENSION: "-debian{{matrix.debianVersion}}-amd64"
+      REPOSITORY: "${STAGING_IMAGE}"
+      DEBIAN_VERSION: "{{matrix.debianVersion}}"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+    matrix:
+      setup:
+        debianVersion:
+          - "10"
+          - "11"
+          - "12"
+
+  - label: ":linux: Build LLVM Apple / Ubuntu ARM - {{matrix.debianVersion}}"
+    key: "build-ubuntu-arm"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
+    command:
+      - ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
+      - ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
+    notify:
+      - github_commit_status:
+          context: "Build LLVM Apple / Ubuntu ARM"
+    env:
+      TAG_EXTENSION: "-debian{{matrix.debianVersion}}-arm64"
+      REPOSITORY: "${STAGING_IMAGE}"
+      DEBIAN_VERSION: "{{matrix.debianVersion}}"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.large"
+    matrix:
+      setup:
+        debianVersion:
+          - "10"
+          - "11"
+          - "12"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -6,7 +6,6 @@ env:
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   DOCKER_REGISTRY: "docker.elastic.co"
   STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
-  DOCKER_REGISTRY_SECRET_PATH: "secret/observability-team/ci/docker-registry/prod"
   BUILDX: 1
 
 steps:
@@ -55,17 +54,14 @@ steps:
 
   - group: "Release"
     key: "release"
-    notify:
-      - github_commit_status:
-          context: "Release / Ubuntu X86_64 && ARM"
 
     steps:
       - label: ":linux: Release / Ubuntu X86_64 - {{matrix.makefile}}"
         key: "release-ubuntu-x86"
         command:
           - ".buildkite/scripts/build.sh {{matrix.makefile}}"
           - ".buildkite/scripts/publish.sh {{matrix.makefile}}"
-        branches: "(main|\\d+\\.\\d+)"
+        if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -79,22 +75,28 @@ steps:
               - "Makefile.debian10"
               - "Makefile.debian11"
               - "Makefile.debian12"
+        notify:
+          - github_commit_status:
+              context: "Release / Ubuntu X86_64"
 
       - label: ":linux: Release / Ubuntu ARM - Makefile.debian9"
         key: "release-ubuntu-arm"
         command:
           - ".buildkite/scripts/build.sh Makefile.debian9"
           - ".buildkite/scripts/publish.sh Makefile.debian9"
-        branches: "(main|\\d+\\.\\d+)"
+        if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
+        notify:
+          - github_commit_status:
+              context: "Release / Ubuntu ARM"
 
       - label: "Post-Release"
         key: "post-release"
         command: ".buildkite/scripts/post-release.sh ${GOLANG_VERSION}"
-        branches: "(main|\\d+\\.\\d+)"
+        if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
         depends_on:
           - "release-ubuntu-x86"
           - "release-ubuntu-arm"
@@ -104,3 +106,4 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+
diff --git a/.buildkite/scripts/changeset/fpm b/.buildkite/scripts/changeset/fpm
@@ -0,0 +1,4 @@
+^\.buildkite/fpm-pipeline.yml$
+^\.buildkite/scripts/llvm-fpm
+^\.buildkite/scripts/changeset/fpm
+^fpm/
diff --git a/.buildkite/scripts/changeset/llvm-apple b/.buildkite/scripts/changeset/llvm-apple
@@ -0,0 +1,4 @@
+^\.buildkite/llvm-apple-pipeline.yml$
+^\.buildkite/scripts/llvm-apple
+^\.buildkite/scripts/changeset/llvm-fpm
+^go/llvm-apple/
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -146,3 +146,15 @@ check_is_arm() {
     is_arm=""
   fi
 }
+
+are_files_changed() {
+  changeset=$1
+
+  if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
+    return 0;
+  else
+    echo "WARN! No files changed in $changeset"
+    return 1;
+  fi
+}
+
diff --git a/.buildkite/scripts/llvm-fpm/build.sh b/.buildkite/scripts/llvm-fpm/build.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+makefile=${1}
+patterns=${2}
+docker_filter_ref=${3}
+
+if ! are_files_changed "$patterns" ; then
+    exit 0
+fi
+
+add_bin_path
+with_go "${GOLANG_VERSION}"
+with_mage
+
+retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
+docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"
diff --git a/.buildkite/scripts/llvm-fpm/publish.sh b/.buildkite/scripts/llvm-fpm/publish.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+makefile=${1}
+patterns=${2}
+
+if ! are_files_changed "$patterns" ; then
+    exit 0
+fi
+
+add_bin_path
+retry 3 make -C ${makefile} push
diff --git a/.buildkite/scripts/post-release.sh b/.buildkite/scripts/post-release.sh
@@ -30,5 +30,3 @@ fi
 set_git_config
 tag_commit
 git_push_with_auth
-
-
diff --git a/catalog-info.yaml b/catalog-info.yaml
@@ -36,14 +36,15 @@ spec:
     spec:
       branch_configuration: "main v1.*" # temporarily disable to build PRs from forks
       pipeline_file: ".buildkite/pipeline.yml"
+      maximum_timeout_in_minutes: 120
       provider_settings:
         build_tags: true
         publish_commit_status: true
         build_pull_request_forks: false
         build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot
         filter_enabled: true
         filter_condition: >-
-          build.creator.name == 'elasticmachine' && build.pull_request.id != null
+          build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null)
       repository: elastic/golang-crossbuild
       cancel_intermediate_builds: true
       cancel_intermediate_builds_branch_filter: '!main'
@@ -79,8 +80,9 @@ spec:
       name: llvm-apple
       description: "Pipeline for LLVM Apple version"
     spec:
-      #      branch_configuration: "main v1.*" # temporarily disable to build PRs from forks  #TODO - enable when pipeline is ready
+      branch_configuration: "main v1.*" # temporarily disable to build PRs from forks
       pipeline_file: ".buildkite/llvm-apple-pipeline.yml"
+      maximum_timeout_in_minutes: 360 # cmake is taking at least 4h to run
       provider_settings:
         build_tags: true
         publish_commit_status: true
@@ -94,8 +96,8 @@ spec:
       cancel_intermediate_builds_branch_filter: '!main'
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: '!main'
-      #      env:
-      #        ELASTIC_PR_COMMENTS_ENABLED: 'true'  #TODO - enable when pipeline is ready
+      env:
+        ELASTIC_PR_COMMENTS_ENABLED: 'true'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
@@ -124,8 +126,9 @@ spec:
       name: fpm
       description: "Pipeline for FPM (packaging made simple)"
     spec:
-      #      branch_configuration: "main v1.*" # temporarily disable to build PRs from forks  #TODO - enable when pipeline is ready
+      branch_configuration: "main v1.*" # temporarily disable to build PRs from forks
       pipeline_file: ".buildkite/fpm-pipeline.yml"
+      maximum_timeout_in_minutes: 120
       provider_settings:
         build_tags: true
         publish_commit_status: true
@@ -139,8 +142,8 @@ spec:
       cancel_intermediate_builds_branch_filter: '!main'
       skip_intermediate_builds: true
       skip_intermediate_builds_branch_filter: '!main'
-      #      env:
-      #        ELASTIC_PR_COMMENTS_ENABLED: 'true'  #TODO - enable when pipeline is ready
+      env:
+        ELASTIC_PR_COMMENTS_ENABLED: 'true'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ

diff --git a/fpm/Dockerfile.tmpl b/fpm/Dockerfile.tmpl
@@ -1,4 +1,4 @@
-FROM debian:buster
+FROM debian:bullseye
 LABEL maintainer="Elastic Beats Team"
 
 RUN \
Original file line number	Diff line number	Diff line change
Expand Up		@@ -30,5 +30,3 @@ fi
		set_git_config
		tag_commit
		git_push_with_auth