Merge branch 'develop' into cameron/spack-fixups

Author: Cameron Rutherford

.github/workflows/pnnl_mirror.yaml

@@ -28,10 +28,10 @@ jobs:
        # Give actions access to some secrets
          with:
             GIT_PUSH_ARGS: '--tags --force --push-option=ci.skip'
-            REMOTE: 'https://gitlab.pnnl.gov/exasgd/frameworks/exago-github-mirror'
+            REMOTE: 'https://gitlab.pnnl.gov/earthshot-exago/exago-github-mirror'
             GIT_USERNAME:  ${{ secrets.GIT_USER }}
             GIT_PASSWORD:  ${{ secrets.GIT_PASSWORD }}
 
       #  Trigger CI pipeline since it was skipped in the above push
        - name: Trigger Pipeline
-         run: curl -X POST -F token=${{ secrets.PNNL_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://gitlab.pnnl.gov/api/v4/projects/1619/trigger/pipeline
+         run: curl -X POST -F token=${{ secrets.PNNL_PIPELINE_TRIGGER }} -F ref=${{ steps.extract_branch.outputs.branch }} https://gitlab.pnnl.gov/api/v4/projects/1719/trigger/pipeline

.gitlab/pnnl/.gitlab-ci.yml

@@ -12,12 +12,13 @@ trigger_newell:
       - local: .gitlab/pnnl/base.gitlab-ci.yml
       - local: .gitlab/pnnl/newell.gitlab-ci.yml
 
-trigger_incline:
-  needs: []
-  trigger:
-    include:
-      - local: .gitlab/pnnl/base.gitlab-ci.yml
-      - local: .gitlab/pnnl/incline.gitlab-ci.yml
+# See - #132 
+#trigger_incline:
+#  needs: []
+#  trigger:
+#    include:
+#      - local: .gitlab/pnnl/base.gitlab-ci.yml
+#      - local: .gitlab/pnnl/incline.gitlab-ci.yml
 
 SVC-Account-Cleanup:
   image: kfox1111/slurm:deception2

.gitlab/pnnl/README.md

@@ -0,0 +1,93 @@
+# How to connect GitHub to a PNNL GitLab (push) Mirror
+
+To run on our HPC clusters at PNNL while hosting our code base on GitHub, we utilize PNNL's CI/CD services on GitLab. 
+
+With GitLab premium, integration between GitHub and GitLab is included. The tier of GitLab that we have does not and did not initially support this. While switching to premium would alleviate some of the burden here, our technical requirements expanded having to test on more than one cluster. This is how we have architected a solution with the base level GitLab offering. 
+
+The PNNL GitLab repository is a push mirror of the GitHub. So whenever a commit is pushed to a pull request or the `main` branch - the changes are pushed to the GitLab and a CI pipeline is triggered.
+
+## Steps in linking the GitLab and GitHub
+
+1. Push mirror & pipeline trigger
+
+In [`pnnl_mirror.yaml`](/.github/workflows/pnnl_mirror.yaml), we use the variables `GIT_USER`, `GIT_PASSWORD`, and `PNNL_PIPELINE_TRIGGER`. `GIT_PASSWORD` and `PNNL_PIPELINE_TRIGGER` are generated in GitLab and then added to GitHub.
+
+`GIT_USER` is the username of whoever will be authenticated when pushing to GitLab for the mirror action. Since we then manually trigger CI after that, and explicitly skip CI here, this name is cosmetic.
+
+a) GIT_PASSWORD
+Go to Settings > Access Tokens. Click `Add new token`.
+
+Pick a reasonable name & expiration date. (ie "GITHUB_PUSH_PASSWORD")
+
+Pick the `Developer` role.
+
+Under `Select scopes`, select `write_repository`.
+
+Create.
+
+Go to GitHub > Settings > Secrets and variables > Actions. Click `New repository secret`.
+Name it `GIT_PASSWORD` and paste in the value generated from GitLab.
+
+Add secret.
+
+b) PNNL_PIPELINE_TRIGGER
+Go to Settings > CI/CD > Pipeline trigger tokens. Click `Add new token`.
+
+Give it a name and click create. 
+
+Go to GitHub > Settings > Secrets and variables > Actions. Click `New repository secret`.
+Name it `PNNL_PIPELINE_TRIGGER` and paste in the value generated from GitLab.
+
+2. Push back status & modules
+
+In our module rebuild pipelines, we rebuild spack modules on each PNNL platform, then commit the new module paths/hashes back to the repository. 
+
+a) Generate token #1 (`GITLAB_MIRROR_STATUS`)
+
+Go to your GitHub profile > Settings > Developer Settings > Personal access tokens > Fine-grained tokens.
+
+Generate new token:
+```
+	resource owner = pnnl
+	select repositories = pnnl/exago
+	permissions > repository permissions > commit statuses (read and write)
+	copy this token
+```
+
+Go to GitLab > Settings > CI/CD > Variables:
+```
+	add variable
+	type = file
+	do not protect/mask/expand
+	key = GITHUB_CURL_HEADER
+	Value = `Authorization: token <token value>`
+```
+
+See https://ecp-ci.gitlab.io/docs/guides/build-status-gitlab.html for more details.
+
+b) Generate token #2 (`GITLAB_COMMIT`)
+
+Go to your GitHub profile > Settings > Developer Settings > Personal access tokens > Fine-grained tokens.
+
+Generate new token: 
+```
+	resource owner = pnnl
+	select repositories = pnnl/exago
+	permissions > repository permissions > contents (read and write)
+	<copy this token>
+```
+
+Go to GitLab > Settings > CI/CD > Variables:
+```
+		add variable
+		type = variable
+		do not protect
+		key = SPACK_GIT_TOKEN
+		<paste in value field>
+```
+
+## Change path to `.gitlab-ci.yml`
+
+Go to > Settings > CI/CD > General Pipelines, change the `CI/CD configuration file` to the correct path to `.gitlab-ci.yml`.
+
+In our repo, the path is `.gitlab/pnnl/.gitlab-ci.yml`.

.gitlab/pnnl/base.gitlab-ci.yml

@@ -21,9 +21,9 @@ include:
       if: '$CI_COMMIT_TITLE =~ /\[deception-rebuild\]/'
       when: never
       # Only run when the commit **DOES NOT** contains "[incline-rebuild]"
-    - &rule_no_incline_rebuild
-      if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/'
-      when: never
+#    - &rule_no_incline_rebuild
+#      if: '$CI_COMMIT_TITLE =~ /\[incline-rebuild\]/'
+#      when: never
       # Only run when the commit **DOES NOT** contains "[ascent-rebuild]" 
     - &rule_no_ascent_rebuild
       if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
@@ -45,13 +45,13 @@ include:
       if: '$CI_COMMIT_TITLE =~ /\[deception-test\]/'
       when: always
     # Only run when the commit **DOES NOT** contains "[incline-test]"
-    - &rule_no_incline_test
-      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
-      when: never
+#    - &rule_no_incline_test
+#      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
+#      when: never
     # Only run when the commit **DOES** contains "[incline-test]"
-    - &rule_yes_incline_test
-      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
-      when: always
+#    - &rule_yes_incline_test
+#      if: '$CI_COMMIT_TITLE =~ /\[incline-test\]/'
+#      when: always
     # Only run when the commit **DOES NOT** contains "[ascent-test]"
     - &rule_no_ascent_test
       if: '$CI_COMMIT_TITLE =~ /\[ascent-test\]/'
@@ -173,7 +173,7 @@ stages:
 
       # jobid used in pnnl_after_script_template to cancel job if cancelled or
       # timed out by gitlab through the UI
-      jobid=$(sbatch --export=ALL -A EXASGD $SLURM_ARGS -p $SLURM_Q -o $output -e $output -t 1:00:00 $WORKDIR/buildsystem/build.sh $SCRIPT_ARGS)
+      jobid=$(sbatch --export=ALL -A earthshot $SLURM_ARGS -p $SLURM_Q -o $output -e $output -t 1:00:00 $WORKDIR/buildsystem/build.sh $SCRIPT_ARGS)
       export jobid=$(echo $jobid | cut -f4 -d' ')
 
       partition=$(squeue -j $jobid -h --format="%P")
@@ -372,8 +372,8 @@ stages:
     - *rule_no_deception_rebuild
     - *rule_no_newell_test
     - *rule_no_newell_rebuild
-    - *rule_no_incline_test
-    - *rule_no_incline_rebuild
+#    - *rule_no_incline_test
+#    - *rule_no_incline_rebuild
     - *rule_no_ascent_test
     - *rule_no_ascent_rebuild
     - *default
@@ -389,8 +389,8 @@ stages:
     - *rule_no_deception_rebuild
     - *rule_yes_newell_test
     - *rule_no_newell_rebuild
-    - *rule_no_incline_test
-    - *rule_no_incline_rebuild
+#    - *rule_no_incline_test
+#    - *rule_no_incline_rebuild
     - *rule_no_ascent_test
     - *rule_no_ascent_rebuild
     - *default
@@ -399,18 +399,18 @@ stages:
     MY_CLUSTER: "newell"
     SLURM_ARGS: " --gres=gpu:1 --ntasks=3 "
 
-.incline:
-  rules:
-    - *rule_no_deception_test
-    - *rule_no_deception_rebuild
-    - *rule_no_newell_test
-    - *rule_no_newell_rebuild
-    - *rule_yes_incline_test
-    - *rule_no_incline_rebuild
-    - *rule_no_ascent_test
-    - *rule_no_ascent_rebuild
-    - *default
-  variables:
-    WORKDIR_SUFFIX: "x86_64-clang-hip-build"
-    MY_CLUSTER: "incline"
-    SLURM_ARGS: " --exclusive --ntasks=3 "
+#.incline:
+#  rules:
+#    - *rule_no_deception_test
+#    - *rule_no_deception_rebuild
+#    - *rule_no_newell_test
+#    - *rule_no_newell_rebuild
+#    - *rule_yes_incline_test
+#    - *rule_no_incline_rebuild
+#    - *rule_no_ascent_test
+#    - *rule_no_ascent_rebuild
+#    - *default
+#  variables:
+#    WORKDIR_SUFFIX: "x86_64-clang-hip-build"
+#    MY_CLUSTER: "incline"
+#    SLURM_ARGS: " --exclusive --ntasks=3 "

buildsystem/spack/binary_mirror.sh

@@ -2,12 +2,12 @@
 
 source buildsystem/spack/load_spack.sh &&
 SPACK_MIRROR="${SPACK_MIRROR:?SPACK_MIRROR is unset. Please use the load_spack script first.}" &&
-spack develop --path=$(pwd) exago@develop &&
-spack bootstrap now &&
-spack concretize -f &&
-(spack mirror create -a --directory $SPACK_MIRROR || true) &&
-spack mirror add local file://$SPACK_MIRROR &&
-spack mirror list
+spack -e $SPACKENV develop --path=$(pwd) exago@develop &&
+spack -e $SPACKENV bootstrap now &&
+spack -e $SPACKENV concretize -f &&
+spack -e $SPACKENV mirror create -a --exclude-specs exago@develop --directory $SPACK_MIRROR &&
+spack -e $SPACKENV mirror add local file://$SPACK_MIRROR &&
+spack -e $SPACKENV mirror list
 
 res=$?
 

buildsystem/spack/configure_modules.sh

@@ -10,15 +10,17 @@ base="./buildsystem/spack/${MY_CLUSTER,,""""}"
 # Printing out loaded modules for debugging...
 module list
 
+echo $SPACKENV
+
 # This assumes that we are installing from a binary mirror, and don't want to fetch files
 # Make sure to use binary_mirror.sh if this is failing
-spack install -j $1 && \
+spack -e $SPACKENV install -j $1 && \
 
 # This deletes the previous modules that are installed
 # Either use a different module path than other users, or deal with frequent updates
 # To use a different module path, you must update the variable $SPACK_MODULES
 # This is configured for the default for the branch in /buildsystem/spack/platform/env.sh
-spack module tcl refresh -y && \
+spack -e $SPACKENV module tcl refresh -y && \
 
 # We will create a new modules file, with the first line being the module path
 mkdir -p $base/modules && \
@@ -35,10 +37,10 @@ echo module use -a $SPACK_INSTALL/$SPACK_MODULES/$arch &> $base/modules/dependen
 echo module use -a $SPACK_INSTALL/$SPACK_MODULES/$arch &> $base/modules/exago.sh && \
 
 # Now we can append to the files
-spack module tcl loads -r -x exago -x openssl exago &>> $base/modules/dependencies.sh && \
+spack -e $SPACKENV module tcl loads -r -x exago -x openssl exago &>> $base/modules/dependencies.sh && \
 # spack module tcl loads -r -x exago -x openssl exago &>> $base/modules/optimized-dependencies.sh && \
 # spack module tcl loads exago &>> $base/modules/exago-optimized.sh && \
-spack module tcl loads exago &>> $base/modules/exago.sh
+spack -e $SPACKENV module tcl loads exago &>> $base/modules/exago.sh
 
 exit_code=$?
 

buildsystem/spack/deception/env.sh

@@ -19,7 +19,7 @@ module load openmpi/4.1.0mlx5.0
 # Define environment variables for where spack stores key files
 # For now, SPACK_INSTALL is the path where everything spack related is installed
 # If you want to modify the module install path, edit the spack.yaml manually
-BASE=/qfs/projects/exasgd/src/deception-ci
+BASE=/qfs/projects/earthshot/src/deception-ci
 export SPACK_INSTALL=$BASE/install
 export SPACK_MODULES=ci-modules
 export SPACK_CACHE=$BASE/../$(whoami)/spack-cache