e2e: add chart upgrade test

.github/workflows/e2e.yaml

@@ -87,14 +87,19 @@ jobs:
       run-uid: ${{ steps.generate-uid.outputs.run-uid }}
       coordinator-digest: ${{ steps.build-coordinator.outputs.digest }}
       update-digest: ${{ steps.build-coordinator-update.outputs.digest }}
+      previous-release-digest: ${{ steps.build-previous-release-coordinator.outputs.digest }}
       marble-digest: ${{ steps.build-marble.outputs.digest }}
       era-config: ${{ steps.build-binaries.outputs.era-config }}
+      previous-release-era-config: ${{ steps.build-previous-release-coordinator.outputs.era-config }}
       marble-config: ${{ steps.build-binaries.outputs.marble-config }}
     runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          fetch-depth: 0
+          fetch-tags: true # Required to check out latest release later
 
       - name: Install dependencies
         env:
@@ -133,13 +138,10 @@ jobs:
       - name: Build binaries
         id: build-binaries
         env:
-          CR: ghcr.io/edgelesssys/marblerun-e2e
-          CONTAINER: coordinator
           COORDINATOR_BUILD_TAGS: ${{ inputs.hsmSealing == 'fake' && 'fakehsm' || '' }}
         run: |
           set -e
           export PATH=${PATH}:/opt/edgelessrt/bin
-          openssl genrsa -out private.pem -3 3072
 
           # Build binaries
           mkdir build && cd build
@@ -210,6 +212,51 @@ jobs:
 
           echo "digest=${digest}" >> $GITHUB_OUTPUT
 
+      - name: Get latest release tag
+        id: get-tag
+        run: |
+          latest_tag=$(git tag --sort=-v:refname | head -n 1)
+          echo "latest_tag=${latest_tag}" >> $GITHUB_OUTPUT
+
+      - name: Checkout latest release
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ steps.get-tag.outputs.latest_tag }}
+          path: previous
+
+      - name: Build Coordinator container from latest release
+        id: build-previous-release-coordinator
+        env:
+          COORDINATOR_BUILD_TAGS: ${{ inputs.hsmSealing == 'fake' && 'fakehsm' || '' }}
+          DOCKER_BUILDKIT: 1
+          CR: ghcr.io/edgelesssys/marblerun-e2e
+          CONTAINER: coordinator
+          UUID: ${{ steps.generate-uid.outputs.run-uid }}
+        working-directory: previous
+        run: |
+          set -e
+          export PATH=${PATH}:/opt/edgelessrt/bin
+
+          # Build binaries
+          mkdir build && cd build
+          cp ${GITHUB_WORKSPACE}/build/private.pem ${GITHUB_WORKSPACE}/build/public.pem ./
+          cmake ..
+          make -j $(nproc) sign-coordinator
+          cp ./coordinator-enclave.signed ./libsymcrypt.so.103 ../
+          # Save era config to output
+          echo "era-config=$(awk '!/UniqueID/' ./coordinator-config.json | base64 -w0)" >> $GITHUB_OUTPUT
+          cd ..
+
+          docker build --tag ${CR}/${CONTAINER}:${UUID}-release -f .github/scripts/coordinator.Dockerfile .
+          docker push ${CR}/${CONTAINER}:${UUID}-release
+
+          # Split image ref into registry/repo/tag and sha256 digest and set in output
+          image_ref=$(docker inspect --format='{{index .RepoDigests 0}}' ${CR}/${CONTAINER}:${UUID}-release)
+          IFS='@' read -ra ADDR <<< "${image_ref}"
+          digest=${ADDR[1]}
+
+          echo "digest=${digest}" >> $GITHUB_OUTPUT
+
   provision-hsm:
     name: Provision HSM for e2e tests
     needs: build-test-containers
@@ -292,6 +339,9 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          fetch-depth: 0
+          fetch-tags: true # Required to check out latest release later
 
       - name: Install dependencies
         run: |
@@ -346,6 +396,25 @@ jobs:
             (.coordinator.hash=\"${{ needs.build-test-containers.outputs.coordinator-digest }}\")" \
             ./charts/values.yaml
 
+      - name: Get latest release tag
+        id: get-tag
+        run: |
+          latest_tag=$(git tag --sort=-v:refname | head -n 1)
+          echo "latest_tag=${latest_tag}" >> $GITHUB_OUTPUT
+
+      - name: Checkout latest release
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          ref: ${{ steps.get-tag.outputs.latest_tag }}
+          path: previous  
+
+      - name: Update latest release MarbleRun chart
+        run: |
+          yq -i eval "(.coordinator.repository=\"$CR\") |
+            (.coordinator.version=\"${{ needs.build-test-containers.outputs.run-uid }}-release\") |
+            (.coordinator.hash=\"${{ needs.build-test-containers.outputs.previous-release-digest }}\")" \
+            ./previous/charts/values.yaml
+
       - name: Run e2e test
         env:
           ERA_CONFIG: ${{ needs.build-test-containers.outputs.era-config }}
@@ -375,6 +444,7 @@ jobs:
             --run "${TEST_FILTER}" \
             --cli /usr/bin/marblerun \
             --chart $PWD/charts/ \
+            --previous-release-chart $PWD/previous/charts/ \
             --kubeconfig $HOME/.kube/config \
             --replicas ${REPLICAS} \
             --era-config $PWD/era.json \

test/e2e/chartupdate_test.go

@@ -0,0 +1,189 @@
+//go:build e2e
+
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: BUSL-1.1
+*/
+
+package main
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	oss_mnf "github.com/edgelesssys/marblerun/coordinator/manifest"
+	"github.com/edgelesssys/marblerun/coordinator/state"
+	"github.com/edgelesssys/marblerun/test/e2e/helm"
+	"github.com/edgelesssys/marblerun/test/e2e/manifest"
+)
+
+func TestChartUpdate(t *testing.T) {
+	if *oldChartPath == "" {
+		t.Skip("No chart path provided to upgrade from. Skipping...")
+	}
+
+	t.Parallel()
+
+	ctx, assert, require, kubectl, cmd, tmpDir := createBaseObjects(t)
+	namespace, releaseName := setUpNamespace(ctx, t, kubectl)
+	getLogsOnFailure(t, kubectl, namespace)
+
+	helm, err := helm.New(t, *kubeConfigPath, namespace)
+	require.NoError(err)
+	t.Logf("Installing chart %q from %q", releaseName, *oldChartPath)
+	uninstall, err := helm.InstallChart(ctx, releaseName, namespace, *oldChartPath, *replicas, defaultTimeout, nil)
+	require.NoError(err)
+	t.Cleanup(uninstall)
+
+	pub, priv := manifest.GenerateKey(t)
+	crt := manifest.GenerateCertificate(t, priv)
+
+	mnf := manifest.DefaultManifest(crt, pub, marbleConfig)
+
+	marble := mnf.Marbles[manifest.DefaultMarble]
+	marbleSecretFile := "/test-secret"
+	previousMarbleSecretFile := marbleSecretFile + ".previous"
+	marble.Parameters.Argv = []string{"marble", "secrets", marbleSecretFile, previousMarbleSecretFile}
+	marble.Parameters.Files[marbleSecretFile] = oss_mnf.File{
+		Data:     "{{ raw .Secrets.ProtectedFilesKey }}",
+		Encoding: "string",
+	}
+	marble.Parameters.Files[previousMarbleSecretFile] = oss_mnf.File{
+		Data:     "{{ raw .Previous.Secrets.ProtectedFilesKey }}",
+		Encoding: "string",
+	}
+	mnf.Marbles[manifest.DefaultMarble] = marble
+
+	manifestPath := writeManifest(t, mnf, tmpDir)
+	getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingManifest)
+	// Set manifest for any Coordinator instance
+	t.Log("Setting manifest")
+	withPortForwardAny(ctx, t, kubectl, namespace, func(port string) error {
+		_, err := cmd.Run(
+			ctx,
+			"manifest", "set",
+			manifestPath, net.JoinHostPort(localhost, port),
+			"--recoverydata", filepath.Join(tmpDir, recoveryDataFile),
+			"--coordinator-cert", filepath.Join(tmpDir, coordinatorCertFileDefault),
+			eraConfig,
+		)
+		return err
+	})
+	t.Log("Manifest set")
+
+	// Verify all instances are accepting marbles
+	getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingMarbles)
+
+	var symmetricKey []byte
+
+	// Set up one HTTP client to use for requests to both the old and new versions of the Coordinator
+	// The new Coordinator instances should successfully reuse the HTTP certificates set up by the old instances
+	coordinatorCertChain, err := os.ReadFile(filepath.Join(tmpDir, coordinatorCertFileDefault))
+	require.NoError(err)
+	caPool := x509.NewCertPool()
+	require.True(caPool.AppendCertsFromPEM(coordinatorCertChain), "failed loading coordinator cert chain")
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: caPool,
+			},
+		},
+	}
+
+	t.Logf("Starting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	podName, err := createMarblePod(ctx, kubectl, namespace, manifest.DefaultMarble, nil, nil)
+	require.NoError(err)
+
+	t.Logf("Checking Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	withPortForward(ctx, t, kubectl, namespace, podName, marbleClientPort, func(port string) error {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s:%s", localhost, port), nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err := client.Do(req)
+		if resp != nil {
+			defer resp.Body.Close()
+		}
+		if err != nil {
+			return err
+		}
+		if http.StatusOK != resp.StatusCode {
+			return fmt.Errorf("http.Get returned: %s", resp.Status)
+		}
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		marbleSecrets := make(map[string][]byte)
+		if err := json.Unmarshal(body, &marbleSecrets); err != nil {
+			return fmt.Errorf("failed to unmarshal response '%s': %w", body, err)
+		}
+		if len(marbleSecrets) != 2 {
+			return fmt.Errorf("expected one secret from Marble, got %d", len(marbleSecrets))
+		}
+		symmetricKey = marbleSecrets[marbleSecretFile]
+		assert.Equal(symmetricKey, marbleSecrets[previousMarbleSecretFile])
+		return nil
+
+	})
+	t.Logf("Deleting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	assert.NoError(kubectl.DeletePod(ctx, namespace, podName))
+
+	// Now upgrade the chart (and Coordinator image) and verify secrets were properly preserved by the Coordinator
+	t.Logf("Upgrading chart %q in namespace %q from %q", releaseName, namespace, *chartPath)
+	require.NoError(helm.UpgradeChart(ctx, releaseName, *chartPath, namespace, defaultTimeout, nil))
+	getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingMarbles)
+
+	t.Logf("Starting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	podName, err = createMarblePod(ctx, kubectl, namespace, manifest.DefaultMarble, nil, nil)
+	require.NoError(err)
+
+	t.Logf("Checking Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	withPortForward(ctx, t, kubectl, namespace, podName, marbleClientPort, func(port string) error {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s:%s", localhost, port), nil)
+		if err != nil {
+			return err
+		}
+
+		resp, err := client.Do(req)
+		if resp != nil {
+			defer resp.Body.Close()
+		}
+		if err != nil {
+			return err
+		}
+		if http.StatusOK != resp.StatusCode {
+			return fmt.Errorf("http.Get returned: %s", resp.Status)
+		}
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		marbleSecrets := make(map[string][]byte)
+		if err := json.Unmarshal(body, &marbleSecrets); err != nil {
+			return fmt.Errorf("failed to unmarshal response '%s': %w", body, err)
+		}
+		if len(marbleSecrets) != 2 {
+			return fmt.Errorf("expected one secret from Marble, got %d", len(marbleSecrets))
+		}
+
+		// Assert the secret wasn't changed by the Coordinator upgrade
+		assert.Equal(symmetricKey, marbleSecrets[marbleSecretFile])
+		assert.Equal(symmetricKey, marbleSecrets[previousMarbleSecretFile])
+		return nil
+
+	})
+
+	t.Logf("Deleting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace)
+	assert.NoError(kubectl.DeletePod(ctx, namespace, podName))
+}

test/e2e/helm/helm.go

@@ -98,6 +98,31 @@ func (h *Helm) InstallChart(
 	return uninstall, nil
 }
 
+// UpgradeChart upgrades an existing MarbleRun installation.
+func (h *Helm) UpgradeChart(
+	ctx context.Context, releaseName, chartPath, namespace string, timeout time.Duration, extraValues map[string]any,
+) error {
+	h.t.Helper()
+
+	upgrade := action.NewUpgrade(h.config)
+	upgrade.Namespace = namespace
+	upgrade.Timeout = timeout
+	upgrade.WaitForJobs = true
+	upgrade.WaitStrategy = kube.StatusWatcherStrategy
+	upgrade.ReuseValues = true // reuse values from install, so we don't have to specify them again
+
+	// Load the chart from the path.
+	chart, err := loader.Load(chartPath)
+	if err != nil {
+		return fmt.Errorf("loading chart: %w", err)
+	}
+
+	if _, err := upgrade.RunWithContext(ctx, releaseName, chart, extraValues); err != nil {
+		return fmt.Errorf("upgrading chart: %w", err)
+	}
+	return nil
+}
+
 func mergeMaps(a, b map[string]any) map[string]any {
 	out := make(map[string]any, len(a))
 	for k, v := range a {

test/e2e/main_test.go

@@ -72,6 +72,7 @@ var (
 	cliPath                      = flag.String("cli", "", "path to MarbleRun CLI")
 	kubeConfigPath               = flag.String("kubeconfig", "", "path to kubeconfig file")
 	chartPath                    = flag.String("chart", "../../charts", "path to helm chart")
+	oldChartPath                 = flag.String("previous-release-chart", "", "path to helm chart of a previous release for upgrade tests")
 	coordinatorUpdateImageSuffix = flag.String("coordinator-update-image-suffix", "", "suffix of the coordinator image to use in update tests")
 	marbleImageName              = flag.String("marble-image-name", "ghcr.io/edgelesssys/marblerun-e2e/test-marble", "name of the marble container image to use in tests")
 	marbleImageVersion           = flag.String("marble-image-version", "latest", "version/tag of the marble container image to use in tests")