diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c87cb9e6f..315988803 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -87,14 +87,19 @@ jobs: run-uid: ${{ steps.generate-uid.outputs.run-uid }} coordinator-digest: ${{ steps.build-coordinator.outputs.digest }} update-digest: ${{ steps.build-coordinator-update.outputs.digest }} + previous-release-digest: ${{ steps.build-previous-release-coordinator.outputs.digest }} marble-digest: ${{ steps.build-marble.outputs.digest }} era-config: ${{ steps.build-binaries.outputs.era-config }} + previous-release-era-config: ${{ steps.build-previous-release-coordinator.outputs.era-config }} marble-config: ${{ steps.build-binaries.outputs.marble-config }} runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + fetch-tags: true # Required to check out latest release later - name: Install dependencies env: @@ -133,13 +138,10 @@ jobs: - name: Build binaries id: build-binaries env: - CR: ghcr.io/edgelesssys/marblerun-e2e - CONTAINER: coordinator COORDINATOR_BUILD_TAGS: ${{ inputs.hsmSealing == 'fake' && 'fakehsm' || '' }} run: | set -e export PATH=${PATH}:/opt/edgelessrt/bin - openssl genrsa -out private.pem -3 3072 # Build binaries mkdir build && cd build @@ -210,6 +212,51 @@ jobs: echo "digest=${digest}" >> $GITHUB_OUTPUT + - name: Get latest release tag + id: get-tag + run: | + latest_tag=$(git tag --sort=-v:refname | head -n 1) + echo "latest_tag=${latest_tag}" >> $GITHUB_OUTPUT + + - name: Checkout latest release + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: ${{ steps.get-tag.outputs.latest_tag }} + path: previous + + - name: Build Coordinator container from latest release + id: build-previous-release-coordinator + env: + COORDINATOR_BUILD_TAGS: ${{ inputs.hsmSealing == 'fake' && 'fakehsm' || '' }} + DOCKER_BUILDKIT: 1 + CR: ghcr.io/edgelesssys/marblerun-e2e + CONTAINER: coordinator + UUID: ${{ steps.generate-uid.outputs.run-uid }} + working-directory: previous + run: | + set -e + export PATH=${PATH}:/opt/edgelessrt/bin + + # Build binaries + mkdir build && cd build + cp ${GITHUB_WORKSPACE}/build/private.pem ${GITHUB_WORKSPACE}/build/public.pem ./ + cmake .. + make -j $(nproc) sign-coordinator + cp ./coordinator-enclave.signed ./libsymcrypt.so.103 ../ + # Save era config to output + echo "era-config=$(awk '!/UniqueID/' ./coordinator-config.json | base64 -w0)" >> $GITHUB_OUTPUT + cd .. + + docker build --tag ${CR}/${CONTAINER}:${UUID}-release -f .github/scripts/coordinator.Dockerfile . + docker push ${CR}/${CONTAINER}:${UUID}-release + + # Split image ref into registry/repo/tag and sha256 digest and set in output + image_ref=$(docker inspect --format='{{index .RepoDigests 0}}' ${CR}/${CONTAINER}:${UUID}-release) + IFS='@' read -ra ADDR <<< "${image_ref}" + digest=${ADDR[1]} + + echo "digest=${digest}" >> $GITHUB_OUTPUT + provision-hsm: name: Provision HSM for e2e tests needs: build-test-containers @@ -292,6 +339,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + fetch-depth: 0 + fetch-tags: true # Required to check out latest release later - name: Install dependencies run: | @@ -346,6 +396,25 @@ jobs: (.coordinator.hash=\"${{ needs.build-test-containers.outputs.coordinator-digest }}\")" \ ./charts/values.yaml + - name: Get latest release tag + id: get-tag + run: | + latest_tag=$(git tag --sort=-v:refname | head -n 1) + echo "latest_tag=${latest_tag}" >> $GITHUB_OUTPUT + + - name: Checkout latest release + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: ${{ steps.get-tag.outputs.latest_tag }} + path: previous + + - name: Update latest release MarbleRun chart + run: | + yq -i eval "(.coordinator.repository=\"$CR\") | + (.coordinator.version=\"${{ needs.build-test-containers.outputs.run-uid }}-release\") | + (.coordinator.hash=\"${{ needs.build-test-containers.outputs.previous-release-digest }}\")" \ + ./previous/charts/values.yaml + - name: Run e2e test env: ERA_CONFIG: ${{ needs.build-test-containers.outputs.era-config }} @@ -375,6 +444,7 @@ jobs: --run "${TEST_FILTER}" \ --cli /usr/bin/marblerun \ --chart $PWD/charts/ \ + --previous-release-chart $PWD/previous/charts/ \ --kubeconfig $HOME/.kube/config \ --replicas ${REPLICAS} \ --era-config $PWD/era.json \ diff --git a/test/e2e/chartupdate_test.go b/test/e2e/chartupdate_test.go new file mode 100644 index 000000000..96fe167f5 --- /dev/null +++ b/test/e2e/chartupdate_test.go @@ -0,0 +1,189 @@ +//go:build e2e + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: BUSL-1.1 +*/ + +package main + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "path/filepath" + "testing" + + oss_mnf "github.com/edgelesssys/marblerun/coordinator/manifest" + "github.com/edgelesssys/marblerun/coordinator/state" + "github.com/edgelesssys/marblerun/test/e2e/helm" + "github.com/edgelesssys/marblerun/test/e2e/manifest" +) + +func TestChartUpdate(t *testing.T) { + if *oldChartPath == "" { + t.Skip("No chart path provided to upgrade from. Skipping...") + } + + t.Parallel() + + ctx, assert, require, kubectl, cmd, tmpDir := createBaseObjects(t) + namespace, releaseName := setUpNamespace(ctx, t, kubectl) + getLogsOnFailure(t, kubectl, namespace) + + helm, err := helm.New(t, *kubeConfigPath, namespace) + require.NoError(err) + t.Logf("Installing chart %q from %q", releaseName, *oldChartPath) + uninstall, err := helm.InstallChart(ctx, releaseName, namespace, *oldChartPath, *replicas, defaultTimeout, nil) + require.NoError(err) + t.Cleanup(uninstall) + + pub, priv := manifest.GenerateKey(t) + crt := manifest.GenerateCertificate(t, priv) + + mnf := manifest.DefaultManifest(crt, pub, marbleConfig) + + marble := mnf.Marbles[manifest.DefaultMarble] + marbleSecretFile := "/test-secret" + previousMarbleSecretFile := marbleSecretFile + ".previous" + marble.Parameters.Argv = []string{"marble", "secrets", marbleSecretFile, previousMarbleSecretFile} + marble.Parameters.Files[marbleSecretFile] = oss_mnf.File{ + Data: "{{ raw .Secrets.ProtectedFilesKey }}", + Encoding: "string", + } + marble.Parameters.Files[previousMarbleSecretFile] = oss_mnf.File{ + Data: "{{ raw .Previous.Secrets.ProtectedFilesKey }}", + Encoding: "string", + } + mnf.Marbles[manifest.DefaultMarble] = marble + + manifestPath := writeManifest(t, mnf, tmpDir) + getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingManifest) + // Set manifest for any Coordinator instance + t.Log("Setting manifest") + withPortForwardAny(ctx, t, kubectl, namespace, func(port string) error { + _, err := cmd.Run( + ctx, + "manifest", "set", + manifestPath, net.JoinHostPort(localhost, port), + "--recoverydata", filepath.Join(tmpDir, recoveryDataFile), + "--coordinator-cert", filepath.Join(tmpDir, coordinatorCertFileDefault), + eraConfig, + ) + return err + }) + t.Log("Manifest set") + + // Verify all instances are accepting marbles + getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingMarbles) + + var symmetricKey []byte + + // Set up one HTTP client to use for requests to both the old and new versions of the Coordinator + // The new Coordinator instances should successfully reuse the HTTP certificates set up by the old instances + coordinatorCertChain, err := os.ReadFile(filepath.Join(tmpDir, coordinatorCertFileDefault)) + require.NoError(err) + caPool := x509.NewCertPool() + require.True(caPool.AppendCertsFromPEM(coordinatorCertChain), "failed loading coordinator cert chain") + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: caPool, + }, + }, + } + + t.Logf("Starting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + podName, err := createMarblePod(ctx, kubectl, namespace, manifest.DefaultMarble, nil, nil) + require.NoError(err) + + t.Logf("Checking Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + withPortForward(ctx, t, kubectl, namespace, podName, marbleClientPort, func(port string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s:%s", localhost, port), nil) + if err != nil { + return err + } + + resp, err := client.Do(req) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + return err + } + if http.StatusOK != resp.StatusCode { + return fmt.Errorf("http.Get returned: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + marbleSecrets := make(map[string][]byte) + if err := json.Unmarshal(body, &marbleSecrets); err != nil { + return fmt.Errorf("failed to unmarshal response '%s': %w", body, err) + } + if len(marbleSecrets) != 2 { + return fmt.Errorf("expected one secret from Marble, got %d", len(marbleSecrets)) + } + symmetricKey = marbleSecrets[marbleSecretFile] + assert.Equal(symmetricKey, marbleSecrets[previousMarbleSecretFile]) + return nil + + }) + t.Logf("Deleting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + assert.NoError(kubectl.DeletePod(ctx, namespace, podName)) + + // Now upgrade the chart (and Coordinator image) and verify secrets were properly preserved by the Coordinator + t.Logf("Upgrading chart %q in namespace %q from %q", releaseName, namespace, *chartPath) + require.NoError(helm.UpgradeChart(ctx, releaseName, *chartPath, namespace, defaultTimeout, nil)) + getStatus(ctx, t, kubectl, cmd, namespace, state.AcceptingMarbles) + + t.Logf("Starting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + podName, err = createMarblePod(ctx, kubectl, namespace, manifest.DefaultMarble, nil, nil) + require.NoError(err) + + t.Logf("Checking Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + withPortForward(ctx, t, kubectl, namespace, podName, marbleClientPort, func(port string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s:%s", localhost, port), nil) + if err != nil { + return err + } + + resp, err := client.Do(req) + if resp != nil { + defer resp.Body.Close() + } + if err != nil { + return err + } + if http.StatusOK != resp.StatusCode { + return fmt.Errorf("http.Get returned: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + marbleSecrets := make(map[string][]byte) + if err := json.Unmarshal(body, &marbleSecrets); err != nil { + return fmt.Errorf("failed to unmarshal response '%s': %w", body, err) + } + if len(marbleSecrets) != 2 { + return fmt.Errorf("expected one secret from Marble, got %d", len(marbleSecrets)) + } + + // Assert the secret wasn't changed by the Coordinator upgrade + assert.Equal(symmetricKey, marbleSecrets[marbleSecretFile]) + assert.Equal(symmetricKey, marbleSecrets[previousMarbleSecretFile]) + return nil + + }) + + t.Logf("Deleting Marble Pod %s in namespace %s", manifest.DefaultMarble, namespace) + assert.NoError(kubectl.DeletePod(ctx, namespace, podName)) +} diff --git a/test/e2e/helm/helm.go b/test/e2e/helm/helm.go index dca7ce57b..25fced9b7 100644 --- a/test/e2e/helm/helm.go +++ b/test/e2e/helm/helm.go @@ -98,6 +98,31 @@ func (h *Helm) InstallChart( return uninstall, nil } +// UpgradeChart upgrades an existing MarbleRun installation. +func (h *Helm) UpgradeChart( + ctx context.Context, releaseName, chartPath, namespace string, timeout time.Duration, extraValues map[string]any, +) error { + h.t.Helper() + + upgrade := action.NewUpgrade(h.config) + upgrade.Namespace = namespace + upgrade.Timeout = timeout + upgrade.WaitForJobs = true + upgrade.WaitStrategy = kube.StatusWatcherStrategy + upgrade.ReuseValues = true // reuse values from install, so we don't have to specify them again + + // Load the chart from the path. + chart, err := loader.Load(chartPath) + if err != nil { + return fmt.Errorf("loading chart: %w", err) + } + + if _, err := upgrade.RunWithContext(ctx, releaseName, chart, extraValues); err != nil { + return fmt.Errorf("upgrading chart: %w", err) + } + return nil +} + func mergeMaps(a, b map[string]any) map[string]any { out := make(map[string]any, len(a)) for k, v := range a { diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index c4c0c9370..44fac30df 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -72,6 +72,7 @@ var ( cliPath = flag.String("cli", "", "path to MarbleRun CLI") kubeConfigPath = flag.String("kubeconfig", "", "path to kubeconfig file") chartPath = flag.String("chart", "../../charts", "path to helm chart") + oldChartPath = flag.String("previous-release-chart", "", "path to helm chart of a previous release for upgrade tests") coordinatorUpdateImageSuffix = flag.String("coordinator-update-image-suffix", "", "suffix of the coordinator image to use in update tests") marbleImageName = flag.String("marble-image-name", "ghcr.io/edgelesssys/marblerun-e2e/test-marble", "name of the marble container image to use in tests") marbleImageVersion = flag.String("marble-image-version", "latest", "version/tag of the marble container image to use in tests")