ci: automate Helm chart packaging and publishing (using Buildkite)

ci/builder/Dockerfile

@@ -382,6 +382,12 @@ RUN curl -fsSL https://releases.hashicorp.com/terraform/1.9.8/terraform_1.9.8_li
     && chmod +x /usr/local/bin/terraform \
     && rm terraform.zip
 
+RUN curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_$ARCH_GO > yq \
+    && if [ $ARCH_GO = amd64 ]; then echo '654d2943ca1d3be2024089eb4f270f4070f491a0610481d128509b2834870049 yq' | sha256sum --check; fi \
+    && if [ $ARCH_GO = arm64 ]; then echo 'ceea73d4c86f2e5c91926ee0639157121f5360da42beeb8357783d79c2cc6a1d yq' | sha256sum --check; fi \
+    && chmod +x yq \
+    && mv yq /usr/local/bin
+
 # Hardcode some known SSH hosts, or else SSH will ask whether the host is
 # trustworthy on the first connection.
 

ci/test/pipeline.template.yml

@@ -158,6 +158,19 @@ steps:
             - exit_status: 1
               limit: 2
 
+      - id: helm-charts-publish
+        label: Publish Helm Charts
+        command: bin/ci-builder run stable misc/helm-charts/publish.sh
+        timeout_in_minutes: 10
+        inputs:
+          - misc/helm-charts
+        depends_on: []
+        agents:
+          queue: linux-aarch64-small
+        branches: main
+        coverage: skip
+        sanitizer: skip
+
   - group: Lints
     key: lints
     steps:

misc/helm-charts/publish.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Copyright Materialize, Inc. and contributors. All rights reserved.
+#
+# Use of this software is governed by the Business Source License
+# included in the LICENSE file at the root of this repository.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0.
+
+set -euo pipefail
+
+. misc/shlib/shlib.bash
+
+CHARTS_DIR=misc/helm-charts
+GITHUB_PAGES_BRANCH=gh-pages
+RELEASE_DIR=.cr-release-packages
+
+# Find directories containing Chart.yaml
+CHARTS=""
+for dir in "$CHARTS_DIR"/*/; do
+  if [ -f "${dir}Chart.yaml" ]; then
+    chart_name=$(basename "$dir")
+    CHARTS="${CHARTS:+${CHARTS} }$chart_name"
+  fi
+done
+if [ -z "$CHARTS" ]; then
+  echo "No valid Helm charts found"
+  exit 0
+fi
+echo "Found valid charts: $CHARTS"
+
+rm -rf gh-pages
+git clone --branch "$GITHUB_PAGES_BRANCH" --depth 1 https://"$GITHUB_TOKEN"@github.com/MaterializeInc/materialize.git gh-pages
+
+mkdir -p $RELEASE_DIR
+CHANGES_MADE=0
+for CHART in $CHARTS; do
+  CHART_PATH="$CHARTS_DIR/$CHART"
+  VERSION=$(yq eval '.version' "$CHART_PATH"/Chart.yaml)
+  echo "Processing chart: $CHART version: $VERSION"
+  # Check if version already exists
+  if [ -f "gh-pages/$CHART-$VERSION.tgz" ]; then
+    echo "Chart $CHART version $VERSION already exists, skipping"
+    continue
+  fi
+  # Lint chart
+  if ! helm lint "$CHART_PATH"; then
+    echo "Linting failed for $CHART"
+    exit 1
+  fi
+  # Package chart
+  helm package "$CHART_PATH" --destination $RELEASE_DIR
+  CHANGES_MADE=1
+done
+# Only proceed if we have new packages
+if [ $CHANGES_MADE -eq 1 ]; then
+  # Copy new charts to gh-pages
+  cp $RELEASE_DIR/*.tgz gh-pages/
+  # Update the repository index
+  cd gh-pages
+  REPO_URL="https://materialize.github.io/materialize"
+  if [ -f index.yaml ]; then
+    helm repo index . --url "$REPO_URL" --merge index.yaml
+  else
+    helm repo index . --url "$REPO_URL"
+  fi
+  # Commit and push changes
+  git add .
+  git config user.email "[email protected]"
+  git config user.name "Buildkite"
+  git commit -m "helm-charts: publish updated charts"
+  git push origin $GITHUB_PAGES_BRANCH
+else
+  echo "No new chart versions to publish"
+fi