Test new ci workflow

.github/actions/sane_workflows/action.yml

@@ -1,39 +1,78 @@
 name: sane-workflows
 description: "Run a set of actions using sane-workflows"
 inputs:
-  archive :
-    required : false
-    type     : string
-    default  : "./"
-
   id:
+    description: ID of workflow run, i.e. name
     required : true
     type     : string
-  host:
+  paths:
+    description: List of paths pointing to workflow(s)
     required : true
+    type     : array
+
+  # Everything else is optional
+  host:
+    description: Specific host to run workflow as
+    required : false
     type     : string
-    default  : ""
+    default  : ''
   actions:
+    description: Actions list to run, mutually exclusive with actions_filter
     required : false
     type     : string
-    default  : ""
+    default  : '[]'
   actions_filter:
+    description: Actions filter to run, mutually exclusive with actions
     required : false
     type     : string
-    default  : ""
+    default  : ''
   args     :
+    description: Additional args to pass to sane_runner
     required : false
     type     : string
-    default  : ""
+    default  : ''
   patch:
+    description: One-time patch dict allowed for modification of workflow, need be
+    required : false
+    type     : string
+    default  : ''
+  upload :
+    description: Whether to upload logs as an artifact on failure
+    required : false
+    type     : boolean
+    default  : false
+
+  # Python environment management
+  cache:
+    description: Whether to cache the python virtual environment
+    required : false
+    type     : boolean
+    default  : false
+  cache_id:
+    description: Specific cache id to use, if not set the key will be inputs.id
+    required : false
+    type     : string
+    default  : ''
+  venv:
+    description: Path to setup python virtual environment at
     required : false
     type     : string
-    default  : ""
+    default  : './venv/sane'
+  python_dependencies:
+    description: Extra python dependencies needed for workflow
+    required : false
+    type     : array
+    default  : '[]'
+  upgrade:
+    description: Use pip install --upgrade when installing dependencies, not recommended with cache
+    required : false
+    type     : boolean
+    default  : true
 
 runs:
   using: "composite"
   steps:
-  - if: ${{ inputs.cache }}
+  - if: ${{ inputs.cache == 'true' }}
     name: Cache python environment
     id: cache-venv
     uses: actions/cache@v3
@@ -42,50 +81,53 @@ runs:
         ${{ inputs.venv }}
       key: venv-${{ inputs.cache_id || inputs.id }}
 
-  - if: ${{ inputs.cache == 'false' || steps.cache-.outputs.cache-hit != 'true' }}
+  - if: ${{ inputs.cache == 'false' || steps.cache-venv.outputs.cache-hit != 'true' }}
     name: Install sane-workflows
     id  : install-sane
+    shell: bash
     run: |
       if [ -x "${{ inputs.venv }}/bin/activate" ]; then
         . ${{ inputs.venv }}/bin/activate
       else
         python3 -m venv ${{ inputs.venv }}
         . ${{ inputs.venv }}/bin/activate
       fi
-      python3 -m pip install sane-workflows
 
-      if [ ! -z "${{ inputs.python_dependencies }}" ]; then
-        python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }}
+      python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} sane-workflows
+
+      if [ "${{ inputs.python_dependencies }}" != "[]" ]; then
+        python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }}
       fi
 
   - name: Run ${{ inputs.id }}
     id  : runTest
+    shell: bash
     run: |
-      NAME=${{ inputs.run_id }}-${{ inputs.id }}
-
       if [ ! -z "${{ inputs.action_filter }}" ]; then
         ACTIONS="-f ${{ inputs.action_filter }}"
-      elif [ ! -z "${{ inputs.actions }}" ]; then
-        ACTIONS="-a   ${{ join( fromJson( inputs.actions ), ' ' ) }}"
+      elif [ "${{ inputs.actions }}" != "[]" ]; then
+        ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}"
       fi
       if [ ! -z "${{ inputs.host }}" ]; then
         HOST="-sh ${{ inputs.host }}"
       fi
       if [ ! -z "${{ inputs.patch }}" ]; then
         mkdir -p .patch-${{ inputs.id }}/
-        echo '{ "patch" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json
+        echo '{ "patches" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json
         PATCH="-p .patch-${{ inputs.id }}/"
       fi
+      . ${{ inputs.venv }}/bin/activate
 
-      sane_runner ${{ input.args }} $ACTIONS $HOST $PATCH \
+      sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \
         -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \
-        -sl .${NAME}_saves \
-        -ll  ${NAME}_logs
+        -sl .${{ inputs.id }}_saves \
+        -ll  ${{ inputs.id }}_logs  \
+        -d
 
   - if  : ${{ inputs.upload && failure() }}
     name: Upload test logs
     uses : actions/upload-artifact@v4
     with:
-      name: ${{ inputs.run_id }}-${{ inputs.id }}_logfiles
-      path: ${{ inputs.run_id }}-${{ inputs.id }}_logs
+      name: ${{ inputs.id }}_logfiles
+      path: ${{ inputs.id }}_logs
       include-hidden-files: true

.github/workflows/ci_rework.yml

@@ -52,98 +52,56 @@ on:
 #   - < next test >
 # https://stackoverflow.com/a/68940067
 jobs:
-  buildcmake:
+  compile-tests:
     if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }}
-    name : Test ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }}
-    runs-on: ${{ matrix.workflow.host }}
-    strategy:
-      max-parallel: 4
-      fail-fast: false
-      matrix:
-        workflow  :
-          - host : derecho
-            name : "Make Compilation Tests"
-            id   : make-tests
-            args : -vr='{"cpus":128,"timelimit":"00:10:00"}'
-            patch : 
-              - priority : -99
-              - hosts :
-                - '[.*derecho.*]' :
-                  - account : NMMM0012
-                  - queue   : main
-            actions :
-              - build_cmake_arw_em_real_gnu_release_dm
-              - build_cmake_arw_em_real_gnu_debug_dm
     # I am leaving this here for posterity if this is to be replicated in private repositories for testing
     permissions:
       contents: read
       pull-requests: write
       statuses: write
-    steps:
-      # Don't use gh checks as they are woefully underdeveloped as a feature leading
-      # to confusing UI and misplaced metrics
-      # https://github.com/orgs/community/discussions/24616
-      - name: Set pending status
-        id: check_run_start
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            github.rest.repos.createCommitStatus({
-              owner:      context.repo.owner,
-              repo:       context.repo.repo,
-              sha:        '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}',
-              target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}',
-              description: '${{ inputs.name }}',
-              context:     '${{ inputs.host }}/${{ inputs.id }}',
-              state:      'pending'
-            })
+    uses : ./.github/workflows/labeled_test.yml
+    name : CMake Compilation Tests
+    with :
+      label    : cmake-compile-tests
+      name     : "CMake Compilation Tests"
+      host     : derecho
+      paths : |
+        [ ".workflow/" ]
+      args : -vr='{"cpus":128,"timelimit":"00:10:00"}'
+      patch: |
+        {
+          "priority" : -99,
+          "hosts" :
+          {
+            "[.*derecho.*]" :
+            {
+              "account" : "NMMM0012",
+              "queue"   : "main",
+              "job_suffix" : "cmake-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}"
+            }
+          }
+        }
+      actions : |
+        [ "build_cmake_arw_em_real_gnu_release_dm",
+          "build_cmake_arw_em_real_gnu_release",
+          "build_cmake_arw_em_real_gnu_debug_dm",
+          "build_cmake_arw_em_b_wave_gnu_debug"
+          ]
 
-      - uses: ./.github/actions/sane-workflows
-        name: ${{ inputs.name }}
-        with:
-          # Everything below this should remain the same and comes from the workflow matrix
-          id       : ${{ matrix.workflow.id }}
-          host     : ${{ matrix.workflow.host }}
-          actions  : ${{ toJson( matrix.workflow.actions ) }}
-          mkdirs   : ${{ matrix.workflow.mkdirs }}
-          args     : ${{ matrix.workflow.args }}
-          patch    : '${{ toJson( matrix.workflow.patch ) }}'
-
-      - name: Set completed status
-        if: ${{ always() }}
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            github.rest.repos.createCommitStatus({
-              owner:        context.repo.owner,
-              repo:         context.repo.repo,
-              sha:          '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}',
-              target_url:   '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}',
-              description:  '${{ inputs.name }}',
-              context:      '${{ inputs.host }}/${{ inputs.id }}',
-              state:        '${{ job.status == 'success' && 'success' || 'failure' }}'
-            })
-
-      - name : Remove '${{ inputs.label }}' label
-        if : ${{ !cancelled() && inputs.event_label == inputs.label }}
-        env:
-          PR_NUMBER: ${{ inputs.event_number }}
-        run: |
-          curl \
-            -X DELETE \
-            -H "Accept: application/vnd.github.v3+json" \
-            -H 'Authorization: token ${{ github.token }}' \
-            https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }}
+      # required to do event trigger
+      event_name   : ${{ inputs.event_name }}
+      event_number : ${{ inputs.event_number }}
+      event_label  : ${{ inputs.test }}
+      ref          : ${{ inputs.ref }}
+      sha          : ${{ inputs.sha }}
 
   # In the event that 'all-tests' is used, this final job will be the one to remove
   # the label from the PR
   removeAllLabel :
     if : ${{ !cancelled() && inputs.event_label == 'all-tests' }}
     name : Remove 'all-tests' label
     runs-on: ubuntu-latest
-    needs : [ buildcmake ] # Put tests here to make this wait for the tests to complete
+    needs : [ compile-tests ] # Put tests here to make this wait for the tests to complete
     permissions:
       pull-requests: write
     steps: