chore: Migrate publishing to GHA, consolidate CI workflows

.github/workflows/daily_ci.yaml

@@ -7,4 +7,4 @@
 
 jobs:
   daily-ci-js-helpers:
-      uses: ./.github/workflows/ci-unit-tests.yaml
+      uses: ./.github/workflows/shared-ci.yml

.github/workflows/prod-release.yml

@@ -0,0 +1,91 @@
+name: Release
+permissions:
+  contents: read
+  id-token: write
+
+on:
+  workflow_dispatch:
+    inputs:
+      version_bump:
+        required: false
+        description: '[Optional] Override semantic versioning with explict version (allowed values: "patch", "minor", "major", or explicit version)'
+        default: ''
+      dist_tag:
+        description: 'NPM distribution tag'
+        required: false
+        default: 'latest'
+      branch:
+        description: 'The branch to release from'
+        required: false
+        default: 'master'
+
+env:
+  NODE_OPTIONS: "--max-old-space-size=4096"
+  NPM_CONFIG_UNSAFE_PERM: true
+
+jobs:
+  pre-release-ci:
+    uses: ./.github/workflows/shared-ci.yml
+
+  # Once all tests have passed, run semantic versioning
+  version:
+    runs-on: ubuntu-latest
+    needs: [pre-release-ci]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Node.js 20
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci --unsafe-perm
+
+      - name: Configure git
+        env:
+          BRANCH: ${{ github.event.inputs.branch }}
+        run: |
+          git config --global user.name "aws-crypto-tools-ci-bot"
+          git config --global user.email "[email protected]"
+          git checkout $BRANCH
+
+      - name: Version packages and push
+        env:
+          VERSION_BUMP: ${{ github.event.inputs.version_bump }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Generate new version and CHANGELOG entry and push it
+          npx lerna version --conventional-commits --git-remote origin --yes ${VERSION_BUMP:+$VERSION_BUMP --force-publish}
+          # Log the commit for posterity
+          git log -n 1
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [pre-release-ci, version]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          registry-url: 'https://registry.npmjs.org'
+
+      # Ensure npm 11.5.1 or later is installed
+      - name: Update npm
+        run: npm install -g npm@latest
+      - run: npm ci --unsafe-perm
+      - run: npm run build --if-present
+      - run: npx lerna publish from-package --yes --dist-tag ${{ github.event.inputs.dist_tag }}
+
+  # Once publishing is complete, validate that the published packages are useable
+  validate:
+    uses: ./.github/workflows/shared-ci.yml
+    needs: [publish]
+    with:
+      test-published-packages: true

.github/workflows/pull.yaml

@@ -1,9 +1,21 @@
 # This workflow runs for every pull request
 name: PR CI
+permissions:
+  contents: read
 
 on:
   pull_request:
 
 jobs:
   pr-ci-js-helpers-test:
-    uses: ./.github/workflows/ci-unit-tests.yaml
+    uses: ./.github/workflows/shared-ci.yml
+  pr-ci-all-required:
+    if: always()
+    needs:
+      - pr-ci-js-helpers-test
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Verify all required jobs passed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}

.github/workflows/push.yaml

@@ -8,4 +8,4 @@
 
 jobs:
   push-ci-js-helpers-test:
-    uses: ./.github/workflows/ci-unit-tests.yaml
+    uses: ./.github/workflows/shared-ci.yml

.github/workflows/shared-ci.yml

@@ -0,0 +1,59 @@
+name: Shared CI Tests
+
+on:
+  workflow_call:
+    inputs:
+      test-published-packages:
+        description: 'Test against published packages instead of checked out code'
+        required: false
+        type: boolean
+        default: false
+
+env:
+  NODE_OPTIONS: "--max-old-space-size=4096"
+  NPM_CONFIG_UNSAFE_PERM: true
+
+jobs:
+  ci-unit-tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        node: ["16.x", "18.x", "20.x"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Node.js ${{ matrix.node }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci --unsafe-perm
+
+      # `npm test` only works for local code, testing published packages requires setup
+      - name: Test local code
+        if: ${{ !inputs.test-published-packages }}
+        run: npm test
+
+      - name: Build (for source code testing)
+        if: ${{ !inputs.test-published-packages }}
+        run: npm run build
+
+      # Run vector tests for all CI runs (Ubuntu only)
+      # Verdaccio is only supported on Node.js v18 and higher
+      # Weird syntax issues on Windows prevent us from running these tests there
+      - name: Publish locally for vector tests (except Node.js 16)
+        if: ${{ !inputs.test-published-packages && matrix.node != '16.x' && matrix.os != 'windows-latest' }}
+        run: npm run verdaccio-publish
+
+      - name: Run vector tests (local packages)
+        if: ${{ !inputs.test-published-packages && matrix.node != '16.x' && matrix.os != 'windows-latest' }}
+        run: npm run verdaccio-verify-publish -- ci
+
+      # Run vector tests against published packages (release workflow validation, Ubuntu only)
+      - name: Run vector tests (published packages)
+        if: ${{ inputs.test-published-packages && matrix.node != '16.x' && matrix.os != 'windows-latest' }}
+        run: npm run verdaccio-verify-publish -- public

util/local_verdaccio_publish

@@ -10,9 +10,7 @@
 // but now the portability problems loom large.
 
 const { spawn, execSync } = require('child_process')
-const { readFileSync } = require('fs')
 const pipeStdIo = { stdio: [process.stdin, process.stdout, process.stderr] }
-const { workspaces } = JSON.parse(readFileSync("package.json", 'utf8'))
 
 // Always clear storage so the latest versions are published
 // I am not worried about _what_ version number is published
@@ -24,6 +22,12 @@ const verdaccio = spawn('npx', ['verdaccio', '-c', 'verdaccio/config.yaml'], pip
   .on('error', e => {
     throw e
   })
+  .on('close', (code, signal) => {
+    console.log(`verdaccio process closed with code ${code} or signal ${signal}`);
+  })
+  .on('exit', (code, signal) => {
+      console.log(`verdaccio process exited with code ${code} or signal ${signal}`);
+  })
 
 // Publish all changed packages the local verdaccio server.
 // Anything that has not been changed will match what is in npm
@@ -37,21 +41,32 @@ const args = [
   '--no-git-reset',
   '--preid', 'ci',
   '--no-verify-access',
-  '--force-publish'
+  '--force-publish',
+  '--loglevel', 'warn',
+  '--no-progress'
 ]
-spawn('npx', args, pipeStdIo)
-  .on('close', (code) => {
-    // Kill the background verdaccio server
-    verdaccio.kill()
-
+timeout = 60000 * 2
+console.log(`Starting lerna publish with timeout of ${timeout}`);
+spawn('npx', args, {
+    stdio: [process.stdin, process.stdout, process.stderr],
+    timeout: timeout
+}).on('close', (code, signal) => {
+    console.log(`lerna terminated due to receipt of signal ${signal} or code ${code}`);
     // The above command will make some modifications,
     // Roll them back
     // Ideally, we would find a way to not have to do this
-    workspaces.forEach(workspace => execSync(`git checkout -- ${workspace}/package.json`))
     execSync('git checkout -- lerna.json')
+    execSync('git restore package-lock.json')
 
+    // Kill the background verdaccio server
+    verdaccioKilledStatus = verdaccio.kill()
+    console.log(`killing Verdaccio returned ${verdaccioKilledStatus}`);
+
     // If this command had an error,
     // we need to forward this.
     // Otherwise the entire CI build may think that things succeeded.
     if (code !== 0) throw Error(`Exit code: ${code}`)
-  })
+
+    process.exit()
+  })
+