Fix release process (#688)

* add pkg/config/config.toml * update documentation * document Make targets for autocomplete * remove obsolete Make target from CI * remove typo from code comment * fix tests that caught invalid toml * pkg/config/config.toml dynamically generated * update docs * ensure dependencies are installed * fix config for both mac and linux * fix ci * don't need the comment line as DevHub strips it anyway * whoops, forgot the cp command * Update DEVELOP.md * support windows * CI * fixing all the things * update deprecated set-output syntax * bump CI actions * bump tinygo
fastly · Oct 21, 2022 · fcd188f · fcd188f
1 parent 3c5384c
commit fcd188f
Show file tree

Hide file tree

Showing 10 changed files with 180 additions and 97 deletions.
diff --git a/.fastly/config.toml b/.fastly/config.toml
@@ -14,5 +14,3 @@ api_endpoint = "https://api.fastly.com"
 
 [viceroy]
 ttl = "24h"
-
-# starter kits will be appended by devhub build process
diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml
@@ -7,25 +7,45 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  config:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@v3
+      - name: "Install Rust"
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable # to install tomlq via `make config`
+      - name: "Generate static app config"
+        run: make config
+      - name: "Config Artifact"
+        uses: actions/upload-artifact@v3
+        with:
+          name: config-artifact-${{ github.sha }}
+          path: pkg/config/config.toml
   lint:
     runs-on: ubuntu-latest
     steps:
     - name: "Checkout code"
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
+    - name: "Install Rust"
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable # to install tomlq via `make vet` pre-requisite
     - name: Install Go
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: 1.18.x
     - name: "Restore golang bin cache"
       id: go-bin-cache
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: ~/go/bin
         key: ${{ runner.os }}-go-bin-${{ hashFiles('~/go/bin') }}-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-bin-
     - name: "Restore golang mod cache"
-      uses: actions/cache@v2
+      uses: actions/cache@v3
       with:
         path: |
           ~/Library/Caches/go-build
@@ -37,13 +57,13 @@ jobs:
       if: steps.go-bin-cache.outputs.cache-hit != 'true'
       run: make dependencies
       shell: bash
-    - name: "Download latest app config"
-      run: |
-        make config
     - name: "Run go mod tidy"
       run: make tidy
     - name: "Run go fmt"
       run: make fmt
+    # NOTE: We don't download the config artifact in this job.
+    # This is because we know Linux is able to generate the configuration file.
+    # Which is triggered by the `make vet` pre-requisite target `config`.
     - name: "Run go vet"
       run: make vet
       shell: bash
@@ -57,60 +77,70 @@ jobs:
       run:  make gosec
       shell: bash
   test:
+    needs: config
     strategy:
       matrix:
-        tinygo-version: [0.24.0]
+        tinygo-version: [0.26.0]
         go-version: [1.18.x]
         node-version: [18]
         rust-toolchain: [stable]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-    - name: "Checkout code"
-      uses: actions/checkout@v2
-    - name: "Install Go"
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-    - uses: Integralist/[email protected]
-      with:
-        tinygo-version: ${{ matrix.tinygo-version }}
-    - name: "Restore golang bin cache"
-      uses: actions/cache@v2
-      with:
-        path: ~/go/bin
-        key: ${{ runner.os }}-go-bin-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-bin-
-    - name: "Restore golang mod cache"
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/Library/Caches/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-mod-
-    - name: "Install Rust"
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust-toolchain }}
-    - name: "Add wasm32-wasi Rust target"
-      run: rustup target add wasm32-wasi --toolchain ${{ matrix.rust-toolchain }}
-    - name: "Validate Rust toolchain"
-      run: rustup show && rustup target list --installed --toolchain stable
-      shell: bash
-    - name: "Install Node"
-      uses: actions/setup-node@v2
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: "Download latest app config"
-      run: |
-        make config
-    - name: "Test suite"
-      run: make test
-      shell: bash
-      env:
-        TEST_COMPUTE_INIT: true
-        TEST_COMPUTE_BUILD: true
-        TEST_COMPUTE_DEPLOY: true
+      - name: "Checkout code"
+        uses: actions/checkout@v3
+      - name: "Install Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: Integralist/[email protected]
+        with:
+          tinygo-version: ${{ matrix.tinygo-version }}
+      - name: "Restore golang bin cache"
+        uses: actions/cache@v3
+        with:
+          path: ~/go/bin
+          key: ${{ runner.os }}-go-bin-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-bin-
+      - name: "Restore golang mod cache"
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/Library/Caches/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-mod-
+      - name: "Install Rust"
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ matrix.rust-toolchain }}
+      - name: "Add wasm32-wasi Rust target"
+        run: rustup target add wasm32-wasi --toolchain ${{ matrix.rust-toolchain }}
+      - name: "Validate Rust toolchain"
+        run: rustup show && rustup target list --installed --toolchain stable
+        shell: bash
+      - name: "Install Node"
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: "Config Artifact"
+        uses: actions/download-artifact@v3
+        with:
+          name: config-artifact-${{ github.sha }}
+      - name: "Move Config"
+        run: mv config.toml pkg/config/config.toml
+      # NOTE: Windows should fail quietly for 'test' pre-requisite target.
+      # On Windows, executing `make config` works fine.
+      # But via GitHub Actions the ../../scripts/config.sh isn't run.
+      # This is because you can't nest PowerShell instances.
+      # Each GitHub Action 'run' step is a PowerShell instance.
+      # And each instance is run as: powershell.exe -command ". '...'"
+      - name: "Test suite"
+        run: make test
+        shell: bash
+        env:
+          TEST_COMPUTE_INIT: true
+          TEST_COMPUTE_BUILD: true
+          TEST_COMPUTE_DEPLOY: true
diff --git a/.github/workflows/tag_release.yml b/.github/workflows/tag_release.yml
@@ -9,22 +9,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: "Checkout code"
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: "Fetch unshallow repo"
       run: git fetch --prune --unshallow
     - name: "Install Go"
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
         go-version: '1.18.x'
     - name: "Set GOHOSTOS and GOHOSTARCH"
       run: echo "GOHOSTOS=$(go env GOHOSTOS)" >> $GITHUB_ENV && echo "GOHOSTARCH=$(go env GOHOSTARCH)" >> $GITHUB_ENV
-    - name: "Download latest app config"
-      run: |
-        make config
-    - name: "Display app config"
-      run: cat pkg/config/config.toml
-    - name: "Validate app config"
-      run: cat pkg/config/config.toml | grep 'remote_config = "https://developer.fastly.com/api/internal/cli-config"'
+    - name: "Install Rust"
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - name: "Generate static app config"
+      run: make config
     # Passing the raw SSH private key causes an error:
     # Load key "/tmp/id_*": invalid format
     #
@@ -38,11 +37,11 @@ jobs:
     - name: "Store AUR_KEY in local file"
       run: echo '${{ secrets.AUR_KEY }}' > '${{ github.workspace }}/aur_key' && chmod 600 '${{ github.workspace }}/aur_key'
     - name: "Run GoReleaser"
-      uses: goreleaser/goreleaser-action@v2
+      uses: goreleaser/goreleaser-action@v3
       with:
         # goreleaser version (NOT goreleaser-action version)
         # update inline with the Makefile
-        version: v1.9.2
+        version: latest
         args: release --rm-dist
       env:
         AUR_KEY: '${{ github.workspace }}/aur_key'

diff --git a/.gitignore b/.gitignore
@@ -63,5 +63,5 @@ vendor/
 aur_key
 
 # Ignore static config that is embedded into the CLI
-# All Makefile targets use the 'config' as a prerequisite (which downloads the config)
+# All Makefile targets use the 'config' as a prerequisite (which generates the config)
 pkg/config/config.toml
diff --git a/DEVELOP.md b/DEVELOP.md
@@ -55,18 +55,12 @@ CLI_CATEGORY=logging CLI_CATEGORY_COMMAND=logging CLI_PACKAGE=foobar CLI_COMMAND
 
 > **NOTE**: Within the generated files, keep an eye out for any `<...>` references that need to be manually updated.
 
-### config.toml
+### `.fastly/config.toml`
 
-When compiling the CLI for a new release, it will pull the required configuration data from the following API endpoint:
+The CLI dynamically generates the `./pkg/config/config.toml` within the CI release process so it can be embedded into the CLI binary. 
 
-```
-https://developer.fastly.com/api/internal/cli-config
-```
-
-The config served from that endpoint is the result of an internal Fastly build process that uses the `.fastly/config.toml` file in the CLI repo as a template, and then dynamically adds in any available Starter Kits.
+The file is added to `.gitignore` to avoid it being added to the git repository.
 
-The configuration is then saved to disk (at the following location) and embedded into the CLI when compiled.
+When compiling the CLI for a new release, it will execute [`./scripts/config.sh`](./scripts/config.sh). The script uses [`./.fastly/config.toml`](./.fastly/config.toml) as a template file to then dynamically inject a list of starter kits (pulling their data from their public repositories).
 
-```
-./pkg/config/config.toml
-```
+The resulting configuration is then saved to disk at `./pkg/config/config.toml` and embedded into the CLI when compiled.
diff --git a/Makefile b/Makefile
@@ -27,8 +27,12 @@ ifeq ($(OS), Windows_NT)
 	.SHELLFLAGS = /c
 	GO_FILES = $(shell where /r pkg *.go)
 	GO_FILES += $(shell where /r cmd *.go)
+	CONFIG_SCRIPT = scripts\config.sh
+	CONFIG_FILE = pkg\config\config.toml
 else
 	GO_FILES = $(shell find cmd pkg -type f -name '*.go')
+	CONFIG_SCRIPT = ./scripts/config.sh
+	CONFIG_FILE = pkg/config/config.toml
 endif
 
 # You can pass flags to goreleaser via GORELEASER_ARGS
@@ -40,69 +44,83 @@ endif
 fastly: dependencies $(GO_FILES)
 	@GOHOSTOS="${GOHOSTOS}" GOHOSTARCH="${GOHOSTARCH}" goreleaser build ${GORELEASER_ARGS}
 
-# useful for attaching a debugger such as https://github.com/go-delve/delve
-debug: config
+# Useful for attaching a debugger such as https://github.com/go-delve/delve
+debug:
 	@$(GO_BIN) build -gcflags="all=-N -l" $(GO_ARGS) -o "fastly" ./cmd/fastly
 
+.PHONY: config
+config:
+	@$(CONFIG_SCRIPT)
+	@cat $(CONFIG_FILE)
+
 .PHONY: all
-all: dependencies config tidy fmt vet staticcheck gosec test build install
+all: config dependencies tidy fmt vet staticcheck gosec test build install
 
-# update goreleaser inline with the release GHA workflow
+# Update CI tools used by ./.github/workflows/pr_test.yml
 .PHONY: dependencies
 dependencies:
 	$(GO_BIN) install github.com/securego/gosec/v2/cmd/gosec@latest
 	$(GO_BIN) install honnef.co/go/tools/cmd/staticcheck@latest
 	$(GO_BIN) install github.com/mgechev/revive@latest
 	$(GO_BIN) install github.com/goreleaser/[email protected]
 
+# Clean up Go modules file.
 .PHONY: tidy
 tidy:
 	$(GO_BIN) mod tidy
 
+# Run formatter.
 .PHONY: fmt
 fmt:
 	@echo gofmt -l ./{cmd,pkg}
 	@eval "bash -c 'F=\$$(gofmt -l ./{cmd,pkg}) ; if [[ \$$F ]] ; then echo \$$F ; exit 1 ; fi'"
 
+# Run static analysis.
 .PHONY: vet
-vet:
+vet: config
 	$(GO_BIN) vet ./{cmd,pkg}/...
 
+# Run linter.
 .PHONY: revive
 revive:
 	revive ./...
 
+# Run security vulnerability checker.
 .PHONY: gosec
 gosec:
 	gosec -quiet -exclude=G104 ./{cmd,pkg}/...
 
+# Run third-party static analysis.
 .PHONY: staticcheck
 staticcheck:
 	staticcheck ./{cmd,pkg}/...
 
+# Run tests
 .PHONY: test
 test: config
 	@$(TEST_COMMAND) -race $(TEST_ARGS)
 
-# GO_ARGS allows for passing additional args to the build e.g.
-# make build GO_ARGS='--ldflags "-s -w"'
+# Compile program.
+#
+# GO_ARGS allows for passing additional arguments.
+# e.g. make build GO_ARGS='--ldflags "-s -w"'
 .PHONY: build
 build: config
 	$(GO_BIN) build $(GO_ARGS) ./cmd/fastly
 
-# GO_ARGS allows for passing additional args to the build e.g.
+# Compile and install program.
+#
+# GO_ARGS allows for passing additional arguments.
 .PHONY: install
 install: config
 	$(GO_BIN) install $(GO_ARGS) ./cmd/fastly
 
-.PHONY: config
-config:
-	@curl -so pkg/config/config.toml https://developer.fastly.com/api/internal/cli-config
-
+# Scaffold a new CLI command from template files.
 .PHONY: scaffold
 scaffold:
 	@$(shell pwd)/scripts/scaffold.sh $(CLI_PACKAGE) $(CLI_COMMAND) $(CLI_API)
 
+# Scaffold a new CLI 'category' command from template files.
 .PHONY: scaffold-category
 scaffold-category:
 	@$(shell pwd)/scripts/scaffold-category.sh $(CLI_CATEGORY) $(CLI_CATEGORY_COMMAND) $(CLI_PACKAGE) $(CLI_COMMAND) $(CLI_API)

diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -305,7 +305,7 @@ func (f *File) SetNonInteractive(v bool) {
 
 // NOTE: Static 👇 is public for the sake of the test suite.
 
-// Static is the embedded configuration file used by the CLI.:was
+// Static is the embedded configuration file used by the CLI.
 //
 //go:embed config.toml
 var Static []byte

diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
@@ -198,7 +198,10 @@ func TestUseStatic(t *testing.T) {
 	// Validate that legacy configuration can be migrated to the static one
 	// embedded in the CLI binary.
 	f := config.File{}
-	f.Read(legacyUserConfigPath, strings.NewReader(""), &out, fsterr.MockLog{}, false)
+	err = f.Read(legacyUserConfigPath, strings.NewReader(""), &out, fsterr.MockLog{}, false)
+	if err != nil {
+		t.Fatalf("unexpected err: %v", err)
+	}
 
 	if f.CLI.Version == "" {
 		t.Fatalf("expected CLI.Version to be set: %+v", f)
Original file line number	Diff line number	Diff line change
Expand Up		@@ -14,5 +14,3 @@ api_endpoint = "https://api.fastly.com"

		[viceroy]
		ttl = "24h"

		# starter kits will be appended by devhub build process