chore: add reusable integ workflow, add publish variables (#2979)

* chore: ADO -> GHA, add more go caching, add docker buildx to jobs, and added reusable workflow for porter integration so each integ gets their own job (stop out of space errors)

Signed-off-by: Sarah Christoff <[email protected]>

---------

Signed-off-by: Sarah Christoff <[email protected]>

.github/workflows/build_pipelinesrelease_template.yml

@@ -92,7 +92,7 @@ jobs:
       shell: bash
   Validate-integration_test:
     env:
-      GHCR_IOGETPORTER_DOCKER_REGISTRY: https://ghcr.io
+      GHCR_IOGETPORTER_DOCKER_REGISTRY: ${{inputs.registry}}
       GHCR_IOGETPORTER_DOCKER_USERNAME: getporterbot
     name: Integration Test
     needs:
@@ -140,7 +140,7 @@ jobs:
       run: go run mage.go ConfigureAgent UseXBuildBinaries
     - name: Run Smoke Tests
       run: mage -v TestSmoke
-  Publish-publish_binaries:
+  publish_binaries:
     name: Publish Binaries
     needs:
     - Validate-build
@@ -168,10 +168,10 @@ jobs:
       env:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       run: mage PublishPorter PublishMixins
-  Publish-publish_docker:
+  publish-ghcr:
     env:
-      DOCKER_REGISTRY:
-      DOCKER_USERNAME:
+      DOCKER_REGISTRY: ${{inputs.registry}}
+      DOCKER_USERNAME: getporterbot
     name: Publish Docker Images
     needs:
     - Validate-build
@@ -195,12 +195,11 @@ jobs:
         path: bin
     - name: Setup Bin
       run: go run mage.go ConfigureAgent UseXBuildBinaries
-    # Unable to determine registry '${{parameters.registry}}' type. The service connection was not found or the authentication type not supported.
-    - name: Docker Login
+    - name: Login to Container Registry
       uses: docker/[email protected]
       with:
-        registry: "${{ env.DOCKER_REGISTRY }}"
-        username: "${{ env.DOCKER_USERNAME }}"
-        password: "${{ secrets.DOCKER_PASSWORD }}"
+        registry: "${{inputs.registry}}"
+        username: "${{ github.actor }}"
+        password: "${{ secrets.GITHUB_TOKEN }}"
     - name: Publish Docker Images to ${{inputs.registry}}
       run: PORTER_REGISTRY=${{inputs.registry}} mage PublishImages

.github/workflows/check-licenses.yaml

@@ -14,7 +14,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true

.github/workflows/close-issues.yaml

@@ -10,10 +10,10 @@ on:
       - release/*
 
 jobs:
-  closeIssue:
+  closeIssueOnPrMergeTrigger:
     runs-on: ubuntu-latest
     steps:
       - name: Closes issues related to a merged pull request.
-        uses: ldez/gha-mjolnir@v1.1.0
+        uses: ldez/gha-mjolnir@v1.0.3
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql.yaml

@@ -21,7 +21,7 @@ jobs:
         with:
           fetch-depth: 0 # Get all git history
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true

.github/workflows/integ-reuseable-workflow.yml

@@ -0,0 +1,41 @@
+name: integ reusable workflow
+
+on:
+  workflow_call:
+    inputs:
+        test_name:
+            type: string
+            required: false
+env:
+    GOVERSION: 1.20.7
+    PORTER_INTEG_FILE: ${{inputs.test_name}}.go
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/[email protected]
+    - uses: actions/setup-go@v4
+      with:
+        go-version: "${{ env.GOVERSION }}"
+        cache: true
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+    - name: Docker Login
+      uses: docker/[email protected]
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    - name: Configure Agent
+      run: go run mage.go build
+      shell: bash
+    - name: Integration Test
+      run: go run mage.go -v TestIntegration
+      shell: bash

.github/workflows/porter-integration.yml

@@ -1,8 +1,4 @@
 name: porter/porter-integration
-# this will only run when this is on main branch apparently so use pr for now (temporary)
-#on:
-#  issue_comment:
-#    types: [created, edited]
 on:
   pull_request:
     branches:
@@ -14,24 +10,114 @@ env:
   GOVERSION: 1.20.7
 
 jobs:
-  Integration_test:
-    name: Integration Test
+  archive_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: archive_test
+  build_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: build_test
+  cli_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: cli_test
+  connection_nix_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: connection_nix_test
+  copy_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: copy_test
+  dependenciesv1_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: dependenciesv1_test
+  dependenciesv2_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: dependenciesv2_test
+  driver_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: driver_test
+  install_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: install_test
+  invoke_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: invoke_test
+  lint_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: lint_test
+  migration_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: migration_test
+  outputs_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: outputs_test
+  publish_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: publish_test
+  pull_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: pull_test
+  registry_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: registry_integration_test
+  schema_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: schema_test
+  sensitive_data_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: sensitive_data_test
+  suppress_output_integration_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: suppress_output_test
+  telemetry_test:
+    uses: getporter/porter/.github/workflows/integ-reuseable-workflow.yml@main
+    with:
+      test_name: telemetry_test
+  # Reusable workflows only supports 20 jobs
+  uninstall_test_integ:
     runs-on: ubuntu-latest
     steps:
     - name: checkout
       uses: actions/[email protected]
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}"
+        cache: true
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
     - name: Docker Login
       uses: docker/[email protected]
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
-    - name: Native Build
-      run: go run mage.go Build
+    - name: Configure Agent
+      run: go run mage.go build
       shell: bash
     - name: Integration Test
-      run: go run mage.go -v TestIntegration
+      env: 
+        PORTER_INTEG_FILE: uninstall_test.go
+      run: go run mage.go -v TestIntegration 
       shell: bash

.github/workflows/porter.yml

@@ -18,6 +18,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Native Build
       run: go run mage.go build
@@ -37,6 +38,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Cross Compile
       run: go run mage.go -v XBuildAll
@@ -56,6 +58,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Unit Test
       run: go run mage.go -v TestUnit
@@ -70,6 +73,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Vet
       run: go run mage.go Vet
@@ -90,6 +94,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Download Cross-Compiled Porter Binaries
       uses: actions/[email protected]
@@ -117,6 +122,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: "${{ env.GOVERSION }}" # The Go version to download (if necessary) and use.
+        cache: true
     - run: go version
     - name: Setup Bin
       run: go run mage.go UseXBuildBinaries

.github/workflows/trivy.yaml

@@ -21,7 +21,7 @@ jobs:
         with:
           fetch-depth: 0 # Get all git history
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
           cache: true

build/testdata/bundles/wordpress/.gitignore

@@ -1,2 +0,0 @@
-Dockerfile
-.cnab

magefile.go

@@ -580,7 +580,15 @@ func TestIntegration() {
 		verbose = "-v"
 	}
 
-	must.Command("go", "test", verbose, "-timeout=30m", run, "-tags=integration", "./...").CollapseArgs().RunV()
+	var path string
+	filename := os.Getenv("PORTER_INTEG_FILE")
+	if filename == "" {
+		path = "./..."
+	} else {
+		path = "./tests/integration/" + filename
+	}
+
+	must.Command("go", "test", verbose, "-timeout=30m", run, "-tags=integration", path).CollapseArgs().RunV()
 }
 
 func TestInitWarnings() {

tests/integration/archive_test.go

@@ -19,7 +19,6 @@ import (
 // Validate that archiving a bundle twice results in the same digest
 func TestArchive_StableDigest(t *testing.T) {
 	t.Parallel()
-
 	p := porter.NewTestPorter(t)
 	defer p.Close()
 	ctx := p.SetupIntegrationTest()

tests/integration/dependenciesv1_test.go

@@ -19,7 +19,6 @@ import (
 
 func TestDependenciesLifecycle(t *testing.T) {
 	t.Parallel()
-
 	p := porter.NewTestPorter(t)
 	defer p.Close()
 	ctx := p.SetupIntegrationTest()

tests/integration/dependenciesv2_test.go

@@ -18,8 +18,8 @@ import (
 
 func TestSharedDependencies(t *testing.T) {
 	t.Parallel()
-
 	p := porter.NewTestPorter(t)
+	defer p.Close()
 	ctx := p.SetupIntegrationTest()
 
 	p.Config.SetExperimentalFlags(experimental.FlagDependenciesV2)