Allow commits as ref for get-spack, protect variable names from colli…

…sions

.gitlab-ci.yml

@@ -15,12 +15,12 @@ variables:
 # at the same location whoever triggers a pipeline.
   CUSTOM_CI_BUILDS_DIR: /usr/workspace/bradiuss/gitlab-runner
 
-  SPACK_PARENT_DIR: ${CI_BUILDS_DIR}/../../llnl-stack-${CI_PIPELINE_ID}
-  SPACK_PATH: ${CI_BUILDS_DIR}/../../llnl-stack-${CI_PIPELINE_ID}/spack
-  SPACK_USER_PATH: /dev/shm/llnl-stack-${CI_PIPELINE_ID}-${CI_JOB_ID}/spack-cache
+  MY_SPACK_PARENT_DIR: ${CI_BUILDS_DIR}/../../llnl-stack-${CI_PIPELINE_ID}
+  MY_SPACK_PATH: ${CI_BUILDS_DIR}/../../llnl-stack-${CI_PIPELINE_ID}/spack
+  MY_SPACK_USER_PATH: /dev/shm/llnl-stack-${CI_PIPELINE_ID}-${CI_JOB_ID}/spack-cache
   #[--create-unique-path]
-  SPACK_DEBUG: "-d"
-  ENV_NAME: ""
+  MY_SPACK_DEBUG: "-d"
+  MY_ENV_NAME: ""
 
 # We only do one generation per stage because we found conflicts otherwise
 #[one-generate-per-stage--]

.gitlab/generate.yml

@@ -9,24 +9,24 @@
 no-env-name:
   extends: [.on-quartz]
   rules:
-    - if: '$ENV_NAME == ""'
+    - if: '$MY_ENV_NAME == ""'
       when: always
     - when: never
   stage: setup
   script:
-    - echo "Variable \"ENV_NAME\" was not set."
-    - echo "\"ENV_NAME\" is required and should point to an existing directory in spack-environments."
+    - echo "Variable \"MY_ENV_NAME\" was not set."
+    - echo "\"MY_ENV_NAME\" is required and should point to an existing directory in spack-environments."
     - exit 1
 
 .env_defined:
   rules:
-    - if: '$ENV_NAME == ""'
+    - if: '$MY_ENV_NAME == ""'
       when: never
     - if: '$CI_JOB_NAME =~ /rm-spack/'
       when: always
     - when: on_success
 
-# Generate CI pipelines for the package in $ENV_NAME
+# Generate CI pipelines for the package in $MY_ENV_NAME
 #[get-spack--]
 get-spack:
   extends: [.on-quartz,.env_defined]
@@ -45,12 +45,12 @@ rm-spack:
 
 .generate-pipeline:
   script:
-    - . ${SPACK_PATH}/share/spack/setup-env.sh
-    - spack env activate spack-environments/${ENV_NAME}
-    - spack ${SPACK_DEBUG} ci generate
+    - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
+    - spack env activate spack-environments/${MY_ENV_NAME}
+    - spack ${MY_SPACK_DEBUG} ci generate
         --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
         --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/pipeline.yml"
-    - cp -r spack-environments/${ENV_NAME}/${SYS_TYPE} ${CI_PROJECT_DIR}/jobs_scratch_dir/concrete_environment/
+    - cp -r spack-environments/${MY_ENV_NAME}/${SYS_TYPE} ${CI_PROJECT_DIR}/jobs_scratch_dir/concrete_environment/
   artifacts:
     paths:
       - "${CI_PROJECT_DIR}/jobs_scratch_dir"

README.md

@@ -12,7 +12,7 @@ Access the [documentation](https://radiuss-spack-testing.readthedocs.io/).
 
 The primary goal of this repo is to be used in Gitlab. The Gitlab CI configuration is such that it will use Spack pipeline feature to generate and run a pipeline that builds one of the environments in the `spack-environments` directory.
 
-The specific environment to be built is controlled by the CI variable `ENV_NAME`.
+The specific environment to be built is controlled by the CI variable `MY_ENV_NAME`.
 
 ### Installing
 

docs/sphinx/add-environment.rst

@@ -45,27 +45,27 @@ CI variables
 Each project must define three (3) environment variables in order for the CI to
 work properly:
 
-* `ENV_NAME`: the name of the environment, which must match one of the
+* `MY_ENV_NAME`: the name of the environment, which must match one of the
   directories under `spack_environments`. This variable is defined in the CI
   YAML configuration, either in `.gitlab-ci.yml` or as a project variable in
   the GitLab instance (web client).
 
 For the other two variables, create a file named `ci-variables.bash` in
-`spack_environments/<ENV_NAME>`. In this file, add the definition of the
+`spack_environments/<MY_ENV_NAME>`. In this file, add the definition of the
 missing variables:
 
-* `SPACK_REPO`: the URL where to clone Spack from.
+* `MY_SPACK_REPO`: the URL where to clone Spack from.
 
-* `SPACK_REF`: the ref to checkout in Spack repo.
+* `MY_SPACK_REF`: the ref to checkout in Spack repo.
 
 .. code-block:: bash
 
-  export SPACK_REPO=https://github.com/spack/spack.git
-  export SPACK_REF=develop
+  export MY_SPACK_REPO=https://github.com/spack/spack.git
+  export MY_SPACK_REF=develop
 
-For now on, each time the CI runs with `ENV_NAME=<env_name>` it will include
+For now on, each time the CI runs with `MY_ENV_NAME=<env_name>` it will include
 the appropriate variable file. All you need to do to run your environment
-pipeline is to set `ENV_NAME`.
+pipeline is to set `MY_ENV_NAME`.
 
 Pull Request worflow
 --------------------
@@ -92,7 +92,7 @@ Getting Started configuration
 -----------------------------
 
 As a getting started recommendation, and if you belong to RADIUSS group, we
-recommend creating a directory with you `ENV_NAME` in:
+recommend creating a directory with you `MY_ENV_NAME` in:
 
 * `/usr/workspace/radiuss/installs/<env_name>`
 * `/usr/workspace/radiuss/mirrors/<env_name>`

docs/sphinx/structure.rst

@@ -41,10 +41,10 @@ global variables defined in ``.gitlab-ci.yml`` are not automatically transmitted
 to the child pipelines.
 
 Among those variables, the user may want to change the value of:
-* ``SPACK_REPO``: the repo used to download Spack
-* ``SPACK_REF``: the ref in Spack repo that will be checked out.
-* ``SPACK_DEBUG``: a placeholder to turn on/off spack debug traces.
-* ``ENV_NAME``: the name of the environment to concretize, build and test.
+* ``MY_SPACK_REPO``: the repo used to download Spack
+* ``MY_SPACK_REF``: the ref in Spack repo that will be checked out.
+* ``MY_SPACK_DEBUG``: a placeholder to turn on/off spack debug traces.
+* ``MY_ENV_NAME``: the name of the environment to concretize, build and test.
 
 To modify any of these variables, it is recommended to use either CI variables
 in Gitlab settings UI, or Pipeline variables found in GitLab pipelines and

scripts/get-spack

@@ -2,22 +2,31 @@
 
 set -e
 
-. spack-environments/${ENV_NAME}/ci-variables.bash
+. spack-environments/${MY_ENV_NAME}/ci-variables.bash
 
 #[get-spack--]
-if [[ ! -d ${SPACK_PATH} ]]
+if [[ ! -d ${MY_SPACK_PATH} ]]
 then
-  mkdir -p ${SPACK_PATH}/..
+  mkdir -p ${MY_SPACK_PATH}/..
   # A shallow clone is enough, and much faster.
-  git clone ${SPACK_REPO} --depth 1 --branch develop ${SPACK_PATH}
+  git clone ${MY_SPACK_REPO} --depth 1 --branch develop ${MY_SPACK_PATH}
 fi
 
-cd ${SPACK_PATH}
-git checkout -b temp
-git branch -D ${SPACK_REF}
-git fetch --depth 1 ${SPACK_REPO} ${SPACK_REF}:${SPACK_REF}
-git checkout ${SPACK_REF}
-git tag ${CI_PIPELINE_ID}
-git branch -D temp
-cd -
+if [[ ${MY_SPACK_COMMIT} ]]
+then # empty spack commit, use MY_SPACK_REF which should have a branch name.
+  cd ${MY_SPACK_PATH}
+  git checkout -b temp
+  git branch -D ${MY_SPACK_REF}
+  git fetch --depth 1 ${MY_SPACK_REPO} ${MY_SPACK_REF}:${MY_SPACK_REF}
+  git checkout ${MY_SPACK_REF}
+  git tag ${CI_PIPELINE_ID}
+  git branch -D temp
+  cd -
+else # MY_SPACK_COMMIT is defined and should have the commit hash to use.
+  cd ${MY_SPACK_PATH}
+  git fetch --depth 1 ${MY_SPACK_REPO} ${MY_SPACK_COMMIT}
+  git checkout ${MY_SPACK_COMMIT}
+  git tag ${CI_PIPELINE_ID}
+  cd -
+fi
 #[--get-spack]

scripts/patch-spack

@@ -1,13 +1,13 @@
 #!/bin/bash
 
-if [[ ! -d ${SPACK_PATH} ]]
+if [[ ! -d ${MY_SPACK_PATH} ]]
 then
-  cd ${SPACK_PATH}
+  cd ${MY_SPACK_PATH}
   git apply ${CI_PROJECT_DIR}/patches/spack/user-scope.patch
   git commit -am "Moving user scope in spack directory"
   cd -
 else
-  echo "\$SPACK_PATH = $SPACK_PATH is not a directory,"
+  echo "\$MY_SPACK_PATH = $MY_SPACK_PATH is not a directory,"
   echo "be sure to run get-spack first."
   exit 1
 fi

scripts/print-variables

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-vars="ENV_NAME SPACK_REF SPACK_REPO SPACK_PATH"
+vars="MY_ENV_NAME MY_SPACK_REF MY_SPACK_COMMIT MY_SPACK_REPO MY_SPACK_PATH"
 
 echo "--------------------------------"
 echo "---- Variables of interest "

scripts/remove-spack

@@ -3,10 +3,10 @@
 set -e
 
 #[rm-spack--]
-if [[ ! -d ${SPACK_PARENT_DIR} ]]
+if [[ ! -d ${MY_SPACK_PARENT_DIR} ]]
 then
   echo "Spack directory not found"
 else
-  rm -rf ${SPACK_PARENT_DIR}
+  rm -rf ${MY_SPACK_PARENT_DIR}
 fi
 #[--rm-spack]

spack-environments/radiuss/ci-variables.bash

@@ -5,5 +5,5 @@
 # SPDX-License-Identifier: (MIT)
 ##############################################################################
 
-export SPACK_REPO=https://github.com/spack/spack.git
-export SPACK_REF=c2af154cd24ab3ef501a7efc020f818ef60aa552
+export MY_SPACK_REPO=https://github.com/spack/spack.git
+export MY_SPACK_COMMIT=c2af154cd24ab3ef501a7efc020f818ef60aa552

spack-environments/radiuss/spack.yaml

@@ -19,10 +19,6 @@ spack:
       padded_length: 128
       projections:
         all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-    'build_stage:':
-    - $spack/var/spack/stage
-  bootstrap:
-    root: /usr/workspace/radiuss/store/bradiuss
   #[--config-override]
 
   mirrors:
@@ -97,7 +93,7 @@ spack:
           tags: [quartz, shell]
       # [--runner-mapping]
           script:
-          - . ${SPACK_PATH}/share/spack/setup-env.sh
+          - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
           - export SPACK_DISABLE_LOCAL_CONFIG=""
           - export SPACK_USER_CACHE_PATH="${SPACK_USER_CACHE}"
           - cd ${SPACK_CONCRETE_ENV_DIR} && spack env activate --without-view .
@@ -109,7 +105,7 @@ spack:
         runner-attributes:
           tags: [lassen, shell]
           script:
-          - . ${SPACK_PATH}/share/spack/setup-env.sh
+          - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
           - export SPACK_DISABLE_LOCAL_CONFIG=""
           - export SPACK_USER_CACHE_PATH="${SPACK_USER_CACHE}"
           - cd ${SPACK_CONCRETE_ENV_DIR} && spack env activate --without-view .
@@ -118,5 +114,5 @@ spack:
     service-job-attributes:
       tags: [quartz, shell]
       before_script:
-      - . ${SPACK_PATH}/share/spack/setup-env.sh
+      - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
 

spack-environments/raja-suite/ci-variables.bash

@@ -5,5 +5,5 @@
 # SPDX-License-Identifier: (MIT)
 ##############################################################################
 
-export SPACK_REPO=https://github.com/davidbeckingsale/spack.git
-export SPACK_REF=update-raja-suite-2022.03.0
+export MY_SPACK_REPO=https://github.com/davidbeckingsale/spack.git
+export MY_SPACK_REF=update-raja-suite-2022.03.0

spack-environments/raja-suite/spack.yaml

@@ -15,15 +15,9 @@ spack:
       padded_length: 128
       projections:
         all: '{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}'
-    misc_cache: $spack/.misc_cache
-    test_stage: $spack/.test_stage
-    'build_stage:':
-    - $spack/var/spack/stage
-  bootstrap:
-    root: /usr/workspace/radiuss/store/bradiuss
   mirrors:
     mirror: file:///usr/workspace/radiuss/mirrors/raja
-  #[definitions--]
+
   definitions:
   - compilers:
       - '%[email protected]'
@@ -102,7 +96,9 @@ spack:
         runner-attributes:
           tags: [quartz, shell]
           script:
-          - . ${SPACK_PATH}/share/spack/setup-env.sh
+          - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
+          - export SPACK_DISABLE_LOCAL_CONFIG=""
+          - export SPACK_USER_CACHE_PATH="${SPACK_USER_CACHE}"
           - cd ${SPACK_CONCRETE_ENV_DIR} && spack env activate --without-view .
           - spack -d ci rebuild
 
@@ -112,11 +108,13 @@ spack:
         runner-attributes:
           tags: [lassen, shell]
           script:
-          - . ${SPACK_PATH}/share/spack/setup-env.sh
+          - . ${MY_SPACK_PATH}/share/spack/setup-env.sh
+          - export SPACK_DISABLE_LOCAL_CONFIG=""
+          - export SPACK_USER_CACHE_PATH="${SPACK_USER_CACHE}"
           - cd ${SPACK_CONCRETE_ENV_DIR} && spack env activate --without-view .
           - spack -d ci rebuild
     service-job-attributes:
       tags: [quartz, shell]
       before_script:
-      - . ${SPACK_PATH}/share/spack/setup-env.sh
+      - . ${MY_SPACK_PATH}/share/spack/setup-env.sh