diff --git a/.github/actions/sane_workflows/action.yml b/.github/actions/sane_workflows/action.yml index b0f09be08c..16913dfbea 100644 --- a/.github/actions/sane_workflows/action.yml +++ b/.github/actions/sane_workflows/action.yml @@ -1,39 +1,78 @@ name: sane-workflows description: "Run a set of actions using sane-workflows" inputs: - archive : - required : false - type : string - default : "./" - id: + description: ID of workflow run, i.e. name required : true type : string - host: + paths: + description: List of paths pointing to workflow(s) required : true + type : array + + # Everything else is optional + host: + description: Specific host to run workflow as + required : false type : string - default : "" + default : '' actions: + description: Actions list to run, mutually exclusive with actions_filter required : false type : string - default : "" + default : '[]' actions_filter: + description: Actions filter to run, mutually exclusive with actions required : false type : string - default : "" + default : '' args : + description: Additional args to pass to sane_runner required : false type : string - default : "" + default : '' patch: + description: One-time patch dict allowed for modification of workflow, need be + required : false + type : string + default : '' + upload : + description: Whether to upload logs as an artifact on failure + required : false + type : boolean + default : false + + # Python environment management + cache: + description: Whether to cache the python virtual environment + required : false + type : boolean + default : false + cache_id: + description: Specific cache id to use, if not set the key will be inputs.id + required : false + type : string + default : '' + venv: + description: Path to setup python virtual environment at required : false type : string - default : "" + default : './venv/sane' + python_dependencies: + description: Extra python dependencies needed for workflow + required : false + type : array + default : '[]' + upgrade: + description: Use pip install --upgrade when installing dependencies, not recommended with cache + required : false + type : boolean + default : true runs: using: "composite" steps: - - if: ${{ inputs.cache }} + - if: ${{ inputs.cache == 'true' }} name: Cache python environment id: cache-venv uses: actions/cache@v3 @@ -42,9 +81,10 @@ runs: ${{ inputs.venv }} key: venv-${{ inputs.cache_id || inputs.id }} - - if: ${{ inputs.cache == 'false' || steps.cache-.outputs.cache-hit != 'true' }} + - if: ${{ inputs.cache == 'false' || steps.cache-venv.outputs.cache-hit != 'true' }} name: Install sane-workflows id : install-sane + shell: bash run: | if [ -x "${{ inputs.venv }}/bin/activate" ]; then . ${{ inputs.venv }}/bin/activate @@ -52,40 +92,42 @@ runs: python3 -m venv ${{ inputs.venv }} . ${{ inputs.venv }}/bin/activate fi - python3 -m pip install sane-workflows - if [ ! -z "${{ inputs.python_dependencies }}" ]; then - python3 -m pip install ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} sane-workflows + + if [ "${{ inputs.python_dependencies }}" != "[]" ]; then + python3 -m pip install --no-cache-dir ${{ inputs.upgrade && '--upgrade' }} ${{ join( fromJson( inputs.python_dependencies ), ' ' ) }} fi - name: Run ${{ inputs.id }} id : runTest + shell: bash run: | - NAME=${{ inputs.run_id }}-${{ inputs.id }} - if [ ! -z "${{ inputs.action_filter }}" ]; then ACTIONS="-f ${{ inputs.action_filter }}" - elif [ ! -z "${{ inputs.actions }}" ]; then - ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" + elif [ "${{ inputs.actions }}" != "[]" ]; then + ACTIONS="-a ${{ join( fromJson( inputs.actions ), ' ' ) }}" fi if [ ! -z "${{ inputs.host }}" ]; then HOST="-sh ${{ inputs.host }}" fi if [ ! -z "${{ inputs.patch }}" ]; then mkdir -p .patch-${{ inputs.id }}/ - echo '{ "patch" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json + echo '{ "patches" : ${{ inputs.patch }} }' >> .patch-${{ inputs.id }}/patch.json PATCH="-p .patch-${{ inputs.id }}/" fi + . ${{ inputs.venv }}/bin/activate - sane_runner ${{ input.args }} $ACTIONS $HOST $PATCH \ + sane_runner ${{ inputs.args }} $ACTIONS $HOST $PATCH \ -p ${{ join( fromJson( inputs.paths ), '-p ' ) }} \ - -sl .${NAME}_saves \ - -ll ${NAME}_logs + -sl .${{ inputs.id }}_saves \ + -ll ${{ inputs.id }}_logs \ + -d - if : ${{ inputs.upload && failure() }} name: Upload test logs uses : actions/upload-artifact@v4 with: - name: ${{ inputs.run_id }}-${{ inputs.id }}_logfiles - path: ${{ inputs.run_id }}-${{ inputs.id }}_logs + name: ${{ inputs.id }}_logfiles + path: ${{ inputs.id }}_logs include-hidden-files: true diff --git a/.github/workflows/ci_rework.yml b/.github/workflows/ci_rework.yml index a39e70065a..5ac90467cd 100644 --- a/.github/workflows/ci_rework.yml +++ b/.github/workflows/ci_rework.yml @@ -52,90 +52,48 @@ on: # - < next test > # https://stackoverflow.com/a/68940067 jobs: - buildcmake: + compile-tests: if : ${{ contains( fromJson('["compile-tests","all-tests"]'), inputs.event_label ) || inputs.event_name == 'push' }} - name : Test ${{ matrix.workflow.name }} on ${{ matrix.workflow.host }} - runs-on: ${{ matrix.workflow.host }} - strategy: - max-parallel: 4 - fail-fast: false - matrix: - workflow : - - host : derecho - name : "Make Compilation Tests" - id : make-tests - args : -vr='{"cpus":128,"timelimit":"00:10:00"}' - patch : - - priority : -99 - - hosts : - - '[.*derecho.*]' : - - account : NMMM0012 - - queue : main - actions : - - build_cmake_arw_em_real_gnu_release_dm - - build_cmake_arw_em_real_gnu_debug_dm # I am leaving this here for posterity if this is to be replicated in private repositories for testing permissions: contents: read pull-requests: write statuses: write - steps: - # Don't use gh checks as they are woefully underdeveloped as a feature leading - # to confusing UI and misplaced metrics - # https://github.com/orgs/community/discussions/24616 - - name: Set pending status - id: check_run_start - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', - target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ inputs.name }}', - context: '${{ inputs.host }}/${{ inputs.id }}', - state: 'pending' - }) + uses : ./.github/workflows/labeled_test.yml + name : CMake Compilation Tests + with : + label : cmake-compile-tests + name : "CMake Compilation Tests" + host : derecho + paths : | + [ ".workflow/" ] + args : -vr='{"cpus":128,"timelimit":"00:10:00"}' + patch: | + { + "priority" : -99, + "hosts" : + { + "[.*derecho.*]" : + { + "account" : "NMMM0012", + "queue" : "main", + "job_suffix" : "cmake-tests.${{ inputs.event_name == 'push' && github.ref_name || inputs.event_number }}" + } + } + } + actions : | + [ "build_cmake_arw_em_real_gnu_release_dm", + "build_cmake_arw_em_real_gnu_release", + "build_cmake_arw_em_real_gnu_debug_dm", + "build_cmake_arw_em_b_wave_gnu_debug" + ] - - uses: ./.github/actions/sane-workflows - name: ${{ inputs.name }} - with: - # Everything below this should remain the same and comes from the workflow matrix - id : ${{ matrix.workflow.id }} - host : ${{ matrix.workflow.host }} - actions : ${{ toJson( matrix.workflow.actions ) }} - mkdirs : ${{ matrix.workflow.mkdirs }} - args : ${{ matrix.workflow.args }} - patch : '${{ toJson( matrix.workflow.patch ) }}' - - - name: Set completed status - if: ${{ always() }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', - target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', - description: '${{ inputs.name }}', - context: '${{ inputs.host }}/${{ inputs.id }}', - state: '${{ job.status == 'success' && 'success' || 'failure' }}' - }) - - - name : Remove '${{ inputs.label }}' label - if : ${{ !cancelled() && inputs.event_label == inputs.label }} - env: - PR_NUMBER: ${{ inputs.event_number }} - run: | - curl \ - -X DELETE \ - -H "Accept: application/vnd.github.v3+json" \ - -H 'Authorization: token ${{ github.token }}' \ - https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + # required to do event trigger + event_name : ${{ inputs.event_name }} + event_number : ${{ inputs.event_number }} + event_label : ${{ inputs.test }} + ref : ${{ inputs.ref }} + sha : ${{ inputs.sha }} # In the event that 'all-tests' is used, this final job will be the one to remove # the label from the PR @@ -143,7 +101,7 @@ jobs: if : ${{ !cancelled() && inputs.event_label == 'all-tests' }} name : Remove 'all-tests' label runs-on: ubuntu-latest - needs : [ buildcmake ] # Put tests here to make this wait for the tests to complete + needs : [ compile-tests ] # Put tests here to make this wait for the tests to complete permissions: pull-requests: write steps: diff --git a/.github/workflows/labeled_test.yml b/.github/workflows/labeled_test.yml new file mode 100644 index 0000000000..d977e2622e --- /dev/null +++ b/.github/workflows/labeled_test.yml @@ -0,0 +1,147 @@ +on : + workflow_call : + inputs : + # Label control + label : + required : true + type : string + name : + required : true + type : string + event_name : + required : true + type : string + event_number: + required : true + type : string + event_label: + required : true + type : string + ref : + required : true + type : string + sha : + required : true + type : string + + # SANE Workflow + paths: + required : true + type : string + host: + required : false + type : string + actions: + required : false + type : string + default : '[]' + actions_filter: + required : false + type : string + args : + required : false + type : string + patch: + required : false + type : string + upload : + required : false + type : boolean + cache: + required : false + type : boolean + cache_id: + required : false + type : string + venv: + required : false + type : string + default : './venv/sane' + python_dependencies: + required : false + type : string + default : '[]' + upgrade: + required : false + type : boolean + default : true + +jobs: + labeled_test: + # Is 5 days a reasonable wait time for testing? + timeout-minutes: 7200 + name: Run ${{ inputs.name }} on ${{ inputs.host }} + runs-on: ${{ inputs.host }} + steps: + # Don't use gh checks as they are woefully underdeveloped as a feature leading + # to confusing UI and misplaced metrics + # https://github.com/orgs/community/discussions/24616 + - name: Set pending status + id: check_run_start + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', + target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', + description: '${{ inputs.name }}', + context: '${{ inputs.host }}/${{ inputs.label }}', + state: 'pending' + }) + + - uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ inputs.event_name == 'push' && github.ref || inputs.ref }} + + - uses: islas/sane-workflows-action@v1.0.0 + name: ${{ inputs.name }} + with: + id : ${{ inputs.label }} + paths : ${{ inputs.paths }} + host : ${{ inputs.host }} + actions : ${{ inputs.actions }} + actions_filter : ${{ inputs.actions_filter }} + args : ${{ inputs.args }} + patch : ${{ inputs.patch }} + upload : ${{ inputs.upload }} + cache : ${{ inputs.cache }} + cache_id : ${{ inputs.cache_id }} + venv : ${{ inputs.venv }} + python_dependencies : ${{ inputs.python_dependencies }} + upgrade : ${{ inputs.upgrade }} + + - name: Set completed status + if: ${{ always() }} + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: '${{ inputs.event_name == 'push' && github.sha || inputs.sha }}', + target_url: '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}', + description: '${{ inputs.name }}', + context: '${{ inputs.host }}/${{ inputs.label }}', + state: '${{ job.status == 'success' && 'success' || 'failure' }}' + }) + + - name : Remove '${{ inputs.label }}' label + if : ${{ !cancelled() && inputs.event_label == inputs.label }} + env: + PR_NUMBER: ${{ inputs.event_number }} + run: | + curl \ + -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ github.token }}' \ + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + + + + + diff --git a/.workflow/builds.py b/.workflow/builds.py index 8e3f13ea40..1a3400008a 100644 --- a/.workflow/builds.py +++ b/.workflow/builds.py @@ -8,14 +8,14 @@ def add_build_for_envs( orch ): dm_opt = [ "ON", "OFF" ] build_types = [ "Release", "Debug" ] configurations = { "ARW" : [ "EM_REAL", "EM_FIRE", "EM_B_WAVE" ] } + orch.log( f"Creating builds for permutations..." ) for core, env, build_type, sm, dm in itertools.product( configurations, envs, build_types, sm_opt, dm_opt ): - orch.log( f"Creating builds for permutation core: {core} env: {env} build_type: {build_type} sm: {sm} dm: {dm}" ) for case in configurations[core]: sm_desc = "_sm" if sm == "ON" else "" dm_desc = "_dm" if dm == "ON" else "" id = f"{core}_{case}_{env}_{build_type}{sm_desc}{dm_desc}".lower() - action = sane.Action( f"build_{id}" ) + action = sane.Action( f"build_cmake_{id}" ) action.config["command"] = ".workflow/scripts/buildCMake.sh" args = []