fix(ci): use cvmfs instead of xrootd, for field maps (#954)

Author: c-dilks

.github/workflows/ci.yml

@@ -42,8 +42,6 @@ jobs:
         with:
           key: raw_test_data # fixed key will always hit; clear cache to trigger cache miss
           path: |
-            clas_005038.evio.00000
-            clas_021559.evio.00001
             clas_018779.evio.00001
           lookup-only: true
       - name: install xrootd-client
@@ -53,10 +51,12 @@ jobs:
           sudo apt -y install xrootd-client
       - name: download
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+#        run: |
+#          xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./
         run: |
-          xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/clas_005038.evio.00000 ./
-          xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-l/clas_021559.evio.00001 ./
-          xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./
+          git lfs install
+          git clone --depth 1 https://code.jlab.org/hallb/clas12/validation-data
+          cp validation-data/raw/rg-d/clas_018779.evio.00001 .
 
   # build
   #############################################################################
@@ -76,14 +76,6 @@ jobs:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
           cache: maven
-      - name: install xrootd-client (linux)
-        if: ${{ matrix.runner == 'ubuntu-latest' }}
-        run: |
-          sudo apt -y update
-          sudo apt -y install xrootd-client
-      - name: install xrootd-client (macos)
-        if: ${{ matrix.runner == 'macos-latest' }}
-        run: brew install xrootd
       - name: setup cvmfs
         uses: cvmfs-contrib/github-action-cvmfs@v5
         with:
@@ -94,7 +86,11 @@ jobs:
         if: ${{ github.ref_type == 'tag' }}
         run: libexec/version-bump.sh ${{ github.ref_name }}
       - name: build
-        run: ./build-coatjava.sh --clara --xrootd --no-progress -T${{ env.nthreads }}
+        run: |
+          git lfs install
+          git submodule update --init ./etc/data/magfield
+          ./build-coatjava.sh --cvmfs --no-progress -T${{ env.nthreads }}
+          ./install-clara -b -c ./coatjava ./clara  
       - name: tar # tarball to preserve permissions
         run: |
           tar czvf coatjava.tar.gz coatjava
@@ -134,15 +130,11 @@ jobs:
           java-version: ${{ matrix.JAVA_VERSION }}
           distribution: ${{ env.java_distribution }}
           cache: maven
-      - name: install xrootd-client
-        run: |
-          sudo apt -y update
-          sudo apt -y install xrootd-client
       - uses: cvmfs-contrib/github-action-cvmfs@v5
         with:
           cvmfs_repositories: 'oasis.opensciencegrid.org'
       - name: unit tests
-        run: ./build-coatjava.sh --xrootd --unittests --no-progress -T${{ env.nthreads }}
+        run: ./build-coatjava.sh --cvmfs --unittests --no-progress -T${{ env.nthreads }}
       - name: collect jacoco report
         if: ${{ matrix.JAVA_VERSION == env.JAVA_VERSION }}
         run: validation/jacoco-aggregate.sh
@@ -190,26 +182,20 @@ jobs:
       - uses: cvmfs-contrib/github-action-cvmfs@v5
         with:
           cvmfs_repositories: 'oasis.opensciencegrid.org'
-      - name: install xrootd-client
-        run: |
-          sudo apt -y update
-          sudo apt -y install xrootd-client
       - uses: actions/download-artifact@v6
         with:
           name: build_ubuntu-latest
       - uses: actions/cache/restore@v4
         with:
           key: raw_test_data
           path: |
-            clas_005038.evio.00000
-            clas_021559.evio.00001
             clas_018779.evio.00001
       - name: untar build
         run: tar xzvf coatjava.tar.gz
       - name: run test
         run: |
           ls -lhtr
-          ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_005038.evio.00000
+          ./coatjava/bin/decoder -n 10000 -o dog.hipo ./clas_018779.evio.00001
 
   test_clara:
     needs: [ build, download_test_data ]
@@ -235,8 +221,6 @@ jobs:
         with:
           key: raw_test_data
           path: |
-            clas_005038.evio.00000
-            clas_021559.evio.00001
             clas_018779.evio.00001
       - name: untar build
         run: |
@@ -287,9 +271,13 @@ jobs:
         with:
           cvmfs_repositories: 'oasis.opensciencegrid.org'
       - name: untar build
-        run: tar xzvf coatjava.tar.gz
+        run: |
+          tar xzvf coatjava.tar.gz
+          tar xzvf clara.tar.gz
       - name: run test
         run: |
+          git lfs install
+          git submodule update --init validation/advanced-tests/data
           cd validation/advanced-tests
           echo "COMMAND: ${{ matrix.cmd }}"
           ${{ matrix.cmd }}

.gitlab-ci.yml

@@ -49,6 +49,8 @@ build:
     - job: mirror
       optional: true
   script:
+    - git lfs install
+    - git submodule update --init etc/data/magfield
     - ./build-coatjava.sh --clara -T$JL_RUNNER_AVAIL_CPU --quiet --no-progress
     - tar -czf coatjava.tar.gz coatjava
     - tar -czf clara.tar.gz clara
@@ -69,7 +71,9 @@ build:
 download:
   stage: build
   script:
-    - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./
+#    - xrdcp xroot://sci-xrootd.jlab.org///osgpool/hallb/clas12/validation/raw/rg-d/clas_018779.evio.00001 ./
+    - git clone https://code.jlab.org/hallb/clas12/validation-data
+    - cp validation-data/raw/rg-d/clas_018779.evio.00001 .
   artifacts:
     when: always
     expire_in: 1 day
@@ -124,6 +128,9 @@ eb:
   dependencies: [build]
   script:
     - tar -xzf coatjava.tar.gz
+    - tar -xzf clara.tar.gz
+    - git lfs install
+    - git submodule update --init validation/advanced-tests/data
     - cd validation/advanced-tests
     - ./run-eb-tests.sh -100 ${ARG}
   parallel:

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "etc/data/magfield"]
+	path = etc/data/magfield
+	url = https://code.jlab.org/hallb/clas12/magfield
+[submodule "validation/advanced-tests/data"]
+	path = validation/advanced-tests/data
+	url = https://code.jlab.org/hallb/clas12/coatjava/advanced-tests

build-coatjava.sh

@@ -23,8 +23,10 @@ usage='''build-coatjava.sh [OPTIONS]... [MAVEN_OPTIONS]...
 
    --xrootd          use xrootd to download field maps
    --cvmfs           use cvmfs to download field maps
+   --lfs             use lfs for field maps and test data
 
    --clara           install clara too
+   --data            download test data (requires lfs)
 
    --help            show this message
 
@@ -40,7 +42,9 @@ downloadMaps="yes"
 runUnitTests="no"
 useXrootd=false
 useCvmfs=false
+useLfs=false
 installClara=false
+downloadData=false
 mvnArgs=()
 wgetArgs=()
 for xx in $@
@@ -61,8 +65,10 @@ do
       wgetArgs+=(--no-verbose)
       ;;
     --xrootd) useXrootd=true ;;
-    --cvmfs) useCvmfs=true ;;
-    --clara) installClara=true ;;
+    --cvmfs)  useCvmfs=true ;;
+    --lfs)    useLfs=true ;;
+    --clara)  installClara=true ;;
+    --data)   downloadData=true ;;
     -h|--help)
       echo "$usage"
       exit 2
@@ -71,6 +77,18 @@ do
   esac
 done
 
+if $downloadData && ! $useLfs; then
+    echo "$usage"
+    echo "ERROR:::::::::::  --data requires --lfs"
+    exit 2
+fi
+
+# Currently only git-lfs works from offsite:
+if ! [[ $(hostname) == *.jlab.org ]]; then
+    echo "INFO:  using --lfs for offsite usage"
+    useLfs=true
+fi
+
 src_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
 prefix_dir=$src_dir/coatjava
 clara_home=$src_dir/clara
@@ -91,8 +109,13 @@ download () {
     if $useXrootd; then
         xrdcp $1 ./
         ret=$?
+    elif $useLfs; then
+        cd $src_dir
+        git submodule update --init etc/data/magfield
+        if $downloadData; then git submodule update --init validation/advanced-tests/data; fi
+        cd - > /dev/null
     elif $useCvmfs; then
-        cp $1 ./
+        cp -v $1 ./
         ret=$?
     elif command_exists wget ; then
         $wget $1
@@ -116,6 +139,7 @@ magfield_dir=$src_dir/etc/data/magfield
 if [ $cleanBuild == "no" ] && [ $downloadMaps == "yes" ]; then
   echo 'Retrieving field maps ...'
   webDir=https://clasweb.jlab.org/clas12offline/magfield
+  if $useLfs; then webDir=${magfield_dir##$src_dir}; fi
   if $useXrootd; then webDir=xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/coatjava/magfield; fi
   if $useCvmfs; then webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield; fi
   mkdir -p $magfield_dir

etc/data/magfield

@@ -56,10 +56,17 @@ function cleanup() {
 }
 
 function build_clara {
-    git clone --branch $clara --depth 1 https://code.jlab.org/hallb/clas12/clara-java
+    #git clone --branch $clara --depth 1 https://code.jlab.org/hallb/clas12/clara-java
+    git clone --depth 1 https://code.jlab.org/hallb/clas12/clara-java
     cd clara-java && CLARA_HOME=$1 ./gradlew deploy && cd -
 }
 
+function build_grapes {
+    #git clone --branch $grapes --depth 1 https://code.jlab.org/hallb/clas12/coatjava/grapes
+    git clone --depth 1 https://code.jlab.org/hallb/clas12/coatjava/grapes
+    cd grapes && mvn package && cd -
+}
+
 trap cleanup EXIT
 
 # Configure:
@@ -133,14 +140,14 @@ fi
 if ! compgen -G "$coatjava/lib/clas/coat-libs-*.jar" > /dev/null
 then
     echo -e "\nRetrieving COATJAVA version $coatjava ..."
-    get https://clasweb.jlab.org/clas12offline/distribution/coatjava/coatjava-$coatjava.tar.gz
-    coatjava=./coatjava
+    get https://github.com/JeffersonLab/coatjava/releases/download/$coatjava/coatjava-$coatjava.tar.gz
+    coatjava=./coatjava-$coatjava
 fi
 mkdir -p clara-cre/plugins/clas12/config
 cp -Lr $coatjava/etc $coatjava/bin $coatjava/lib* clara-cre/plugins/clas12
 
 # GRAPES:
 echo -e "\nRetrieving GRAPES version $grapes ..."
-get https://clasweb.jlab.org/clas12offline/distribution/grapes/grapes-$grapes.tar.gz
-mv grapes-$grapes clara-cre/plugins/grapes
-
+build_grapes $PWD/grapes
+mkdir -p clara-cre/plugins/grapes/lib/core/
+cp grapes/target/grapes-*core*.jar clara-cre/plugins/grapes/lib/core/

install-clara

@@ -1,27 +1,24 @@
 #!/bin/bash -f
 
-# coatjava must already be built at ../../coatjava/
+# coatjava and clara must already be built at ../../coatjava/
+# and input data files at ./data
 
 # set up environment
 JAVA_OPTS="-Djava.util.logging.config.file=$PWD/../../etc/logging/debug.properties"
-CLARA_HOME=$PWD/clara_installation/ ; export CLARA_HOME
+CLARA_HOME=$PWD/../../clara/ ; export CLARA_HOME
 COAT=$CLARA_HOME/plugins/clas12/
 
-# install clara
-../../install-clara -c ../../coatjava $CLARA_HOME
-[ $? -ne 0 ] && echo "clara installation error" && exit 1
-
 # source coatjava environment
 source $COAT/libexec/env.sh
 classPath="${COATJAVA_CLASSPATH}:../lib/*:src/"
 
 # download test files
-wget --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/twoTrackEvents_809_raw.evio.tar.gz
-[ $? -ne 0 ] && echo "wget validation files failure" && exit 2
-tar -zxvf twoTrackEvents_809_raw.evio.tar.gz
+#wget --no-check-certificate http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/twoTrackEvents_809_raw.evio.tar.gz
+#[ $? -ne 0 ] && echo "wget validation files failure" && exit 2
+#tar -zxvf twoTrackEvents_809_raw.evio.tar.gz
 
 # run decoder
-$COAT/bin/decoder -t -0.5 -s 0.0 -i ./twoTrackEvents_809_raw.evio -o ./twoTrackEvents_809.hipo -c 2
+$COAT/bin/decoder -t -0.5 -s 0.0 -i ./data/twoTrackEvents_809_raw.evio -o ./twoTrackEvents_809.hipo -c 2
 [ $? -ne 0 ] && echo "decoder failure" && exit 3
 
 # take a peek

validation/advanced-tests/data

@@ -3,8 +3,10 @@
 webDir=http://clasweb.jlab.org/clas12offline/distribution/coatjava/validation_files/eb
 webVersion=5.11-fid-tm-dc2-r11
 webDir=$webDir/$webVersion
+webDir=./data/$webVersion
 
 # coatjava must already be built at ../../coatjava/
+# and input test data files at ./data
 
 # whether to use CLARA (0=no)
 useClara=0
@@ -121,8 +123,8 @@ then
     fi
 
     # download test files, if necessary:
-    wget -N --no-check-certificate $webDir/${stub}.hipo
-    if [ $? != 0 ] ; then echo "wget validation files failure" ; exit 1 ; fi
+    #wget -N --no-check-certificate $webDir/${stub}.hipo
+    #if [ $? != 0 ] ; then echo "wget validation files failure" ; exit 1 ; fi
 
     # update the schema dictionary:  (no longer necessary now that recon-util does it)
     #rm -f up_${stub}.hipo
@@ -134,7 +136,7 @@ then
     then
         GEOMDBVAR=$geoDbVariation
         export GEOMDBVAR
-        ../../coatjava/bin/recon-util -i ${stub}.hipo -o out_${stub}.hipo -c 2
+        ../../coatjava/bin/recon-util -i ${webDir}/${stub}.hipo -o out_${stub}.hipo -c 2
     else
         echo "set inputDir $PWD/" > cook.clara
         echo "set outputDir $PWD/" >> cook.clara