3.1 release -> master

.github/lychee.toml

@@ -0,0 +1,12 @@
+# This lychee configuration tries to find dead links in zwe, pswi, and smpe files which may be displayed to the user.
+#   This does not focus on dead links for internal-facing documentation, e.g. README.md.
+
+# We're including language sources as well as document files since the language sources embed URLs
+#  which can be displayed to the user, e.g. 'zwe' shell script commands displaying help information.
+
+exclude_path = ['smpe/bld/service/_README_sample_sysmod_builds.docx']
+# This exclude list covers known urls which should be ignored, including ones which contained 
+#  templated values in typescript. The templates are covered by a wildcard.
+exclude = [ 'https://www.eclipse.org/legal/epl-v20.html', 'https://zowe.org/schemas/*', 'https://www.zowe.org/schemas/*', 'https://discovery', 'https://github.com/zowe/launcher/blob/*', 'https://myregistry/url$', 'https://your-zos-host-name/zosmf/*{1,60}', '^http://service.software.ibm.com/holdata/390holddata.html',  ]
+
+hidden = true

.github/scripts/cicd_test/make_matrix.sh

@@ -32,15 +32,14 @@ case $install_test_choice in
   test_file="$KEYRING_TESTFILE"
   ;;
 
-"z/OS node v18")
-  test_file="$ZOS_NODE_V18_TESTFILE"
-  test_force_system="zzow08"
-  ;;
-
 "z/OS node v20")
   test_file="$ZOS_NODE_V20_TESTFILE"
   ;;
 
+"z/OS node v22")
+  test_file="$ZOS_NODE_V22_TESTFILE"
+  ;;
+
 "Non-strict Verify External Certificate")
   test_file="$NON_STRICT_VERIFY_EXTERNAL_CERTIFICATE_TESTFILE"
   ;;
@@ -87,18 +86,18 @@ if [[ ! -z "$test_force_system" ]]; then
 else
   if [[ -z "$dont_parse_test_server" ]]; then
     if [[ "$test_server" == "Any zzow servers" ]]; then
-      test_server="zzow0"$(echo $(($RANDOM % 3 + 6)))
+      test_server="zzow"$(printf %02d $(($RANDOM % 3 + 9)))
     fi
     TEST_FILE_SERVER="$test_file($test_server)"
   else
     any_occurrence=$(echo $test_file | grep -o "(any)" | wc -l)
     interim_test_file_server=$test_file
     for i in $(seq $any_occurrence); do
-      # Generates zzow06, zzow07, zzow08
-      interim_test_file_server=$(echo $interim_test_file_server | sed "s#(any)#(zzow0$(echo $(($RANDOM % 3 + 6))))#")
+      # Generates zzow09, zzow10, zzow11
+      interim_test_file_server=$(echo $interim_test_file_server | sed "s#(any)#(zzow$(printf %02d $(($RANDOM % 3 + 9))))#")
     done
 
-    TEST_FILE_SERVER=$(echo $interim_test_file_server | sed "s#(all)#(zzow06,zzow07,zzow08)#g")
+    TEST_FILE_SERVER=$(echo $interim_test_file_server | sed "s#(all)#(zzow09,zzow10,zzow11)#g")
   fi
 fi
 

.github/scripts/cicd_test/prep7_more_test_prep/03_process_test_server.sh

@@ -14,16 +14,16 @@ TEST_SERVER=$(echo "$MATRIX_SERVER" | cut -d "-" -f2)
 
 case $TEST_SERVER in
 
-"zzow06")
-  TEST_SERVER_NICKNAME=marist-6
+"zzow09")
+  TEST_SERVER_NICKNAME=marist-9
   ;;
 
-"zzow07")
-  TEST_SERVER_NICKNAME=marist-7
+"zzow10")
+  TEST_SERVER_NICKNAME=marist-10
   ;;
 
-"zzow08")
-  TEST_SERVER_NICKNAME=marist-8
+"zzow11")
+  TEST_SERVER_NICKNAME=marist-11
   ;;
 
 *)

.github/scripts/cicd_test/prep7_more_test_prep/05_process_ext_list.sh

@@ -35,11 +35,12 @@ if [[ "$MATRIX_TEST" == *"install-ext"* ]]; then
         if [[ "$each_ext" == *"("* ]] && [[ "$each_ext" == *")"* ]] ; then
             # user provides custom artifactory pattern
             ext_name=$(echo "$each_ext" | cut -d "(" -f1)
-            ext_pattern=$(echo "$each_ext" | cut -d "(" -f2 | cut -d ")" -f1)
+            ext_version=$(echo "$each_ext" | cut -d "(" -f2 | cut -d ")" -f1)
+            ext_pattern=$(echo "$DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN" | sed "s#{ext-name}#$ext_name#g" | sed "s#{ext-version}#$ext_version#g" )
         else
             # use default
             ext_name="$each_ext"
-            ext_pattern=$(echo "$DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN" | sed "s#{ext-name}#$ext_name#g")
+            ext_pattern=$(echo "$DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN" | sed "s#{ext-name}#$ext_name#g" | sed "s#{ext-version}#*#g" )
         fi
 
         echo "[Check 4 INFO] extension name is $ext_name"
@@ -58,4 +59,4 @@ if [[ "$MATRIX_TEST" == *"install-ext"* ]]; then
     EXTENSION_LIST=$(echo $EXTENSION_LIST | sed 's/;$//g')
     assert_env_var EXTENSION_LIST
     printf "${GREEN}[Check 4/$TOTAL_CHECK] Zowe extension list processing complete!${NC}\n"
-fi
+fi

.github/scripts/cicd_test/prep7_more_test_prep/06_process_outputs.sh

@@ -13,7 +13,7 @@
 # Set outputs
 echo ZOWE_ARTIFACTORY_FINAL=$ZOWE_ARTIFACTORY_FINAL >>$GITHUB_OUTPUT
 echo ZOWE_ARTIFACTORY_FINAL_FILENAME=$ZOWE_ARTIFACTORY_FINAL_FILENAME >>$GITHUB_OUTPUT
-echo NODE_HOME_PATTERN=/ZOWE/node/node-$ZOS_NODE_VERSION-os390-s390x >>$GITHUB_OUTPUT
+echo NODE_HOME_PATTERN=/ZOWE/node/node-$ZOS_NODE_VERSION >>$GITHUB_OUTPUT
 echo TEST_SERVER=$TEST_SERVER >>$GITHUB_OUTPUT
 echo TEST_SERVER_NICKNAME=$TEST_SERVER_NICKNAME >>$GITHUB_OUTPUT
 echo EXTENSION_LIST=$EXTENSION_LIST >>$GITHUB_OUTPUT
@@ -28,4 +28,4 @@ printf "Zowe CLI artifactory path: ${CYAN}$ZOWE_CLI_ARTIFACTORY_FINAL${NC}\n"
 printf "Zowe extension list: ${CYAN}$EXTENSION_LIST${NC}\n"
 printf "Test server: ${CYAN}$TEST_SERVER${NC}\n"
 printf "Test server nickname: ${CYAN}$TEST_SERVER_NICKNAME${NC}\n"
-printf "Node home pattern on z/OS: ${CYAN}/ZOWE/node/node-$ZOS_NODE_VERSION-os390-s390x${NC}\n"
+printf "Node home pattern on z/OS: ${CYAN}/ZOWE/node/node-$ZOS_NODE_VERSION${NC}\n"

.github/workflows/build-packaging.yml

@@ -99,8 +99,9 @@ jobs:
           echo "run_build=${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.ORIGIN_ISSUE_TRIGGER == 'false') || !contains(fromJson(steps.get-labels.outputs.result), 'Build: None') }}" >> $GITHUB_OUTPUT
       - id: check-test
         name: 'export conditional used to determine if we should run a test suite'
+        # run_test explanation: if we set "run_build" from the prior step and this is a PR, run unless "Test: None" label is present. If not initiated from a PR, only build.
         run: |
-          echo "run_test=${{ (steps.check-build.outputs.run_build == 'true' && !contains(fromJson(steps.get-labels.outputs.result), 'Test: None')) }}" >> $GITHUB_OUTPUT
+          echo "run_test=${{ (steps.check-build.outputs.run_build == 'true' && github.event_name != 'workflow_dispatch' && !contains(fromJson(steps.get-labels.outputs.result), 'Test: None')) }}" >> $GITHUB_OUTPUT
 
   display-dispatch-event-id:
     if: github.event.inputs.RANDOM_DISPATCH_EVENT_ID != ''
@@ -344,13 +345,13 @@ jobs:
         with:
           lock-repository: ${{ github.repository }}
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          lock-resource-name: zowe-psi-build-zzow07-lock
+          lock-resource-name: zowe-psi-build-zzow10-lock
           lock-avg-retry-interval: 60
 
       - name: '[PSWI 0] PSWI pre-build check for existing smpe'
         if: env.INPUTS_BUILD_PSWI == 'true' && github.event.inputs.PSWI_SMPE_ARTIFACTORY_PATH != '' && github.event.inputs.PSWI_SMPE_AZWE_ARTIFACTORY_PATH != ''
         run: |
-          jfrog rt dl ${{github.event.inputs.PSWI_SMPE_AZWE_ARTIFACTORY_PATH}}/AZWE002*.zip --flat=true .pax/AZWE002.zip
+          jfrog rt dl ${{github.event.inputs.PSWI_SMPE_AZWE_ARTIFACTORY_PATH}}/AZWE003*.zip --flat=true .pax/AZWE003.zip
           jfrog rt dl ${{github.event.inputs.PSWI_SMPE_ARTIFACTORY_PATH}}/zowe-smpe-*.zip --flat=true .pax/zowe-smpe.zip
 
       - name: '[SMPE Pax 4] Build PSWI'
@@ -373,7 +374,7 @@ jobs:
         with:
           name: pswi-folder
           path: |
-            pswi/**
+            pswi/logs/**
 
       - name: '[K8S] Build Kubernetes'
         timeout-minutes: 10

.github/workflows/cicd-test-readme.md

@@ -4,9 +4,9 @@ This guide will describe how you should input into Github Actions workflow input
 
 Currently we support three testing z/OS servers:
 
-- zzow06 (ACF2)
-- zzow07 (Top Secret/TSS)
-- zzow08 (RACF)
+- zzow09 (ACF2)
+- zzow10 (Top Secret/TSS)
+- zzow11 (RACF)
 
 Testing pipeline is running tests in parallel. The workflow will try to acquire the resource lock if available. If the resource lock is occupied, the workflow will wait until the lock is succesfully acquired.
 
@@ -17,7 +17,7 @@ Workflow trigger is at [cicd-test](https://github.com/zowe/zowe-install-packagin
 ### Choose Test Server
 
 - This input is a choice, and it's mandatory.  
-- You can choose from one of `zzow06`, `zzow07`, `zzow08`, `zzow06,zzow07,zzow08` (if you want to run the test on all zzow servers), or `Any zzow servers` (pick any zzow servers, potentially help reduce wait time)
+- You can choose from one of `zzow09`, `zzow10`, `zzow11`, `zzow09,zzow10,zzow11` (if you want to run the test on all zzow servers), or `Any zzow servers` (pick any zzow servers, potentially help reduce wait time)
 - Default is `Any zzow servers`
 
 ### Choose Install Test
@@ -29,7 +29,7 @@ Workflow trigger is at [cicd-test](https://github.com/zowe/zowe-install-packagin
   - SMPE PTF
   - Extensions
   - Keyring
-  - z/OS node v18
+  - z/OS node v20
   - Non-strict Verify External Certificate
   - Install PTF twice
   - VSAM Caching storage method
@@ -91,7 +91,7 @@ Background: CICD testing relies on a `zowe.pax` or `zowe-smpe.zip` (for SMPE ins
 
 - This input is pre-filled with `sample-node-api;sample-trial-app` to test [sample-node-api](https://github.com/zowe/sample-node-api) and [sample-trial-app](https://github.com/zowe/sample-trial-app) projects. In normal circumstances, you probably don't need to modify the pre-filled value here.
 - By default, the extension artifact search pattern is using format `libs-snapshot-local/org/zowe/{ext-name}/*/{ext-name}-*.pax` where `{ext-name}` will be processed and substituted from this input (as an example above, `sample-node-api`). Then the latest uploaded artifact will be used.
-- Optionally, you can customized your extension artifact path. Customized jfrog artifactory path should exist, be valid, and enclosed in brackets and put after the extension name, eg. `sample-node-api(my/new/path/sample-node-api-cus.pax)`. A pattern contains `*` is also supported, which the latest artifact will be picked up. If multiple extensions are included, make sure to separate them by semi-colon. In addition to the artifactory path/pattern, you can also put a full http URL to any other remote location that points to an extension pax here.
+- Optionally, you can customized your extension artifact version. Customized jfrog artifactory version should exist, be valid, and enclosed in brackets and put after the extension name, eg. `sample-node-api(3.0.0-SNAPSHOT)`. This example will create a search pattern like the following, where the latest artifact in the folder is picked up: `libs-snapshot-local/org/zowe/sample-node-api/3.0.0-SNAPSHOT/sample-node-api-*.pax`.  A pattern containing `*` is also supported, which will result in the latest artifact in the latest folder matching the pattern. For example: `sample-node-api(1.0.0-*)` will create this search pattern: `libs-snapshot-local/org/zowe/sample-node-api/1.0.0-*/sample-node-api-*.pax`, which could match folders `1.0.0-SNAPSHOT`, `1.0.0-MAIN`, `1.0.0-user-pr-build`, etc.
 - The following regular expression will be used to check against your input
 
   ```
@@ -109,7 +109,7 @@ Background: CICD testing relies on a `zowe.pax` or `zowe-smpe.zip` (for SMPE ins
 When running CICD integration tests during RC stage, the following string will be parsed into the Github Actions matrix. As a result, a total of 21 independent jobs will be spawned.
 
 ```
-basic/install.ts(zzow06,zzow07,zzow08);basic/install-ptf.ts(zzow06,zzow07,zzow08);basic/install-ext.ts(zzow07);extended/keyring.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v18.ts(zzow06,zzow07,zzow08);extended/node-versions/node-v20.ts(zzow06,zzow07,zzow08);extended/certificates/nonstrict-verify-external-certificate.ts(zzow06)
+basic/install.ts(zzow09,zzow10,zzow11);basic/install-ptf.ts(zzow09,zzow10,zzow11);basic/install-ext.ts(zzow10);extended/keyring.ts(zzow09,zzow10,zzow11);extended/node-versions/node-v20.ts(zzow09,zzow10,zzow11);extended/node-versions/node-v20.ts(zzow09,zzow10,zzow11);extended/certificates/nonstrict-verify-external-certificate.ts(zzow09)
 ```
 
 Total elapsed time when running in parallel is approximately 3.5 hours on paper idealy if all parallel jobs are executing at the same time. In reality, from numerous tests performed, total elapsed time is around 4 hours.  
@@ -121,7 +121,6 @@ Selected test running elapsed time:
 | ---- | ------------ |
 | Convenience Pax | 53m |
 | SMPE PTF | 68m |
-| z/OS node v18 | 45m |
 | z/OS node v20 | 45m |
 | Keyring | 53m |
 | Non-strict Verify External Certificate | 51m |

.github/workflows/cicd-test.yml

@@ -16,10 +16,10 @@ on:
         default: 'Any zzow servers'
         options: 
           - Any zzow servers
-          - zzow06
-          - zzow07
-          - zzow08
-          - zzow06,zzow07,zzow08
+          - zzow09
+          - zzow10
+          - zzow11
+          - zzow09,zzow10,zzow11
       install-test:
         description: 'Choose Install Test'
         type: choice
@@ -31,8 +31,8 @@ on:
           - SMPE PTF
           - Extensions
           - Keyring
-          - z/OS node v18
           - z/OS node v20
+          - z/OS node v22
           - Non-strict Verify External Certificate
           - Install PTF Twice
           - VSAM Caching Storage Method
@@ -52,7 +52,7 @@ on:
         required: false
         # FIXME: too slow to test 2, temporarily only test 1
         # default: 'sample-node-api;sample-trial-app'
-        default: 'sample-node-api'
+        default: 'sample-node-api(3.0.0-SNAPSHOT)'
       RANDOM_DISPATCH_EVENT_ID:
         description: 'random dispatch event id'
         required: false
@@ -65,12 +65,12 @@ env:
   SANITY_TEST_PATH: tests/sanity
   DEFAULT_ZOWE_PAX_ARTIFACTORY_PATTERN: libs-snapshot-local/org/zowe/*zowe*{branch-name}*.pax
   DEFAULT_ZOWE_SMPE_ARTIFACTORY_PATTERN: libs-snapshot-local/org/zowe/*zowe-smpe*{branch-name}*.zip
-  DEFAULT_ZOWE_CLI_ARTIFACTORY_PATTERN: PLACE_HOLDER/org/zowe/cli/zowe-cli-package/*zowe-cli-package-2*.zip
-  DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN: libs-snapshot-local/org/zowe/{ext-name}/*/{ext-name}-*.pax
+  DEFAULT_ZOWE_CLI_ARTIFACTORY_PATTERN: PLACE_HOLDER/org/zowe/cli/zowe-cli-package/*zowe-cli-package-3*.zip
+  DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN: libs-snapshot-local/org/zowe/{ext-name}/{ext-version}/{ext-name}-*.pax
 
   # below block can be overwritten, adjusted by DevOps only
-  # ZOS_NODE_VERSION more to choose from: v16.20.1,  v18.16.0
-  ZOS_NODE_VERSION: v16.20.1
+  # ZOS_NODE_VERSION more to choose from: v18.16.0, v20.11.0
+  ZOS_NODE_VERSION: v18.16.0
   CLIENT_NODE_VERSION: v12.18.3
   INSTALL_TEST_DEBUG_INFORMATION: zowe-install-test:*
   SANITY_TEST_DEBUG_INFORMATION: zowe-sanity-test:*
@@ -81,16 +81,16 @@ env:
   SMPE_PTF_TESTFILE: basic/install-ptf.ts
   EXTENSIONS_TESTFILE: basic/install-ext.ts
   KEYRING_TESTFILE: extended/keyring.ts
-  ZOS_NODE_V18_TESTFILE: extended/node-versions/node-v18.ts
   ZOS_NODE_V20_TESTFILE: extended/node-versions/node-v20.ts
+  ZOS_NODE_V22_TESTFILE: extended/node-versions/node-v22.ts
   NON_STRICT_VERIFY_EXTERNAL_CERTIFICATE_TESTFILE: extended/certificates/nonstrict-verify-external-certificate.ts
   INSTALL_PTF_TWICE_TESTFILE: extended/install-ptf-two-times.ts
   VSAM_CACHING_STORAGE_METHOD_TESTFILE: extended/caching-storages/vsam-storage.ts
   INFINISPAN_CACHING_STORAGE_METHOD_TESTFILE: extended/caching-storages/infinispan-storage.ts
   CONFIG_MANAGER_TESTFILE: extended/config-manager/enable-config-manager.ts
   GENERAL_API_DOCUMENTATION_TESTFILE: basic/install-api-gen.ts
-  ZOWE_NIGHTLY_TESTS_FULL: basic/install.ts(all);basic/install-fmid.ts(all)
-  ZOWE_RELEASE_TESTS_FULL: basic/install.ts(all);basic/install-fmid.ts(all);basic/install-ext.ts(any);extended/keyring.ts(all);extended/node-versions/node-v18.ts(zzow08):extended/certificates/nonstrict-verify-external-certificate.ts(any);extended/caching-storages/infinispan-storage.ts(any);extended/config-manager/enable-config-manager.ts(any)
+  ZOWE_NIGHTLY_TESTS_FULL: basic/install.ts(all);basic/install-ptf.ts(all)
+  ZOWE_RELEASE_TESTS_FULL: basic/install.ts(all);basic/install-ptf.ts(all);basic/install-ext.ts(any);extended/keyring.ts(all);extended/node-versions/node-v20.ts(any);extended/node-versions/node-v22.ts(any);extended/certificates/nonstrict-verify-external-certificate.ts(any);extended/caching-storages/infinispan-storage.ts(any);extended/config-manager/enable-config-manager.ts(any)
 
 jobs:
   display-dispatch-event-id:
@@ -298,9 +298,6 @@ jobs:
               case 'Keyring':
                 eta = 53
                 break;
-              case 'z/OS node v18':
-                eta = 45
-                break;
               case 'z/OS node v20':
                 eta = 45
                 break;
@@ -385,7 +382,7 @@ jobs:
 
       - name: '[After Test 2] Upload test report'
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: InstallTestReports-${{ env.TEST_NAME }}-${{ steps.more-test-prep.outputs.TEST_SERVER_NICKNAME }}-${{ github.run_id }}-${{ env.CURRENT_TIME }}
           path: ${{ env.INSTALL_TEST_PATH }}/reports/

.github/workflows/diff-schema.yml

@@ -34,7 +34,7 @@ jobs:
       - name: '[Build] Make diff'
         run: git diff ${{ env.FROM }} ${{ env.TO }} -- schemas > schemas.diff
       - name: '[Upload]'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: schemas.diff
           path: schemas.diff

.github/workflows/diff-yaml

@@ -34,7 +34,7 @@ jobs:
       - name: '[Build] Make diff'
         run: git diff ${{ env.FROM }} ${{ env.TO }} -- example-zowe.yaml > example-yaml.diff
       - name: '[Upload]'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: example-yaml.diff
           path: example-yaml.diff

.github/workflows/link-check.yml

@@ -0,0 +1,49 @@
+# This link checker focuses on covering end-user facing links, not developer documentation.
+#   This means we cover the smp/e, pswi, and zwe content.
+# The lychee toml files are set to ignore links which popped up in comments or variables.
+permissions: read-all
+
+name: Link Check CI
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+  push:
+    branches:
+    - 'v3.x/staging'
+    - 'v2.x/staging'
+  workflow_dispatch:
+
+jobs:
+  lychee-link-check:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    - name: Link Checker
+      id: lychee
+      uses: lycheeverse/lychee-action@f81112d0d2814ded911bd23e3beaa9dda9093915 # v2.1.0
+      with:
+        fail: true
+        format: json
+        output: lychee_output.json
+        args: --config ./.github/lychee.toml --no-progress --verbose --max-retries 10 'bin/**/*' 'smpe/**/*' 'workflows/**/*'
+
+    - name: Output failed links in console
+      id: quick-output
+      if: always()
+      run: |
+        if [[ -f "lychee_output.json" ]]; then
+          jq .fail_map lychee_output.json
+        else
+          echo "lychee command failed and didn't produce output"
+        fi
+
+    - name: Publish Output
+      id: publish
+      if: always()
+      uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882
+      with: 
+        name: lychee_results.json
+        path: lychee_output.json