NDCMS · btovar · Jan 28, 2025
diff --git a/apptainer-images/distributions/al8/spec.sin b/apptainer-images/distributions/al8/spec.sin
@@ -0,0 +1,89 @@
+BootStrap: docker
+From: gitlab-registry.cern.ch/linuxsupport/alma8-base:latest
+
+%labels
+    maintainer "CMS Data Preservation and Open Access Group <[email protected]>"
+    org.label-schema.build-date $BUILD_DATE \
+    org.label-schema.name "AlmaLinux 8 CMS base OS" \
+    org.label-schema.description "AlmaLinux8 OS capable of using/running CMS software release(s)." \
+    org.label-schema.url "http://cms-sw.github.io/" \
+    org.label-schema.vcs-ref $VCS_REF \
+    org.label-schema.vcs-url $VCS_URL \
+    org.label-schema.vendor "CERN" \
+    org.label-schema.version $VERSION \
+    org.label-schema.schema-version "1.0"
+
+%files
+    ../../dotfiles/dot-pythonrc.py /etc/pythonrc.py
+    ../../dotfiles/dot-bashrc /etc/profile.d/bashrc.sh
+    ../../entrypoints/entrypoint-nd.sh /opt/cms/entrypoint.sh
+    cvmfsexec-mnt /opt/cms/cvmfsexec-mnt
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat8/cctools/current/bin/work_queue_worker /usr/bin/work_queue_worker
+
+    # parrot from redhat7, it is ok because the binary is statically linked.
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/lobster-171-cd5e3e2c-cvmfs-70dfa0d6/bin/parrot_cvmfs_static_run /usr/bin/parrot_run
+
+%environment
+    export LC_ALL=en_US.UTF-8
+    export LANG=en_US.UTF-8
+    export USER=cmsusr
+    export HOME=/home/cmsusr
+
+%post
+        yum install -y libX11-devel libXext-devel mesa-libGLU-devel \
+        mesa-libGL-devel libSM libXft libXext \
+        pciutils glx-utils mesa-dri-drivers libX11 libXi libXrender \
+        tcsh zsh tcl tk e2fsprogs perl-ExtUtils-Embed libXmu e2fsprogs-libs libXpm bc libaio \
+        tar patch krb5-devel perl-Data-Dumper perl-CGI perl-DBI perl-YAML gcc unzip zip perl-libwww-perl libXpm-devel libXft-devel svn \
+        gcc-c++ strace cern-wrappers krb5-workstation wget hostname readline-devel nano bzip2 perl-Switch perl-Storable \
+        perl-Env perl-Thread-Queue CERN-CA-certs tk-devel tcl-devel which \
+        java-1.8.0-openjdk java-1.8.0-openjdk-devel popt popt-devel libXcursor libXrandr libXinerama nspr nss nss-util nss-devel file file-libs \
+        readline bzip2-libs libgfortran time \
+        git \
+        openssl \
+        glibc-devel.i686 glibc-devel \
+        glibc-headers \
+        sudo nano && \
+        yum clean -y all
+
+        wget http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo && \
+        mv egi-trustanchors.repo /etc/yum.repos.d/ && \
+        wget http://repository.egi.eu/sw/production/cas/1/GPG-KEY-EUGridPMA-RPM-3 && \
+        mv GPG-KEY-EUGridPMA-RPM-3 /etc/pki/rpm-gpg/ && \
+        wget http://linuxsoft.cern.ch/wlcg/wlcg-centos8.repo && \
+        mv wlcg-centos8.repo /etc/yum.repos.d/ && \
+        wget http://linuxsoft.cern.ch/wlcg/RPM-GPG-KEY-wlcg && \
+        mv RPM-GPG-KEY-wlcg /etc/pki/rpm-gpg/ && \
+        yum install -y ca-policy-egi-core wlcg-repo.noarch wlcg-iam-lsc-cms HEP_OSlibs && \
+        yum install -y glibc-langpack-en glibc-locale-source
+	    mkdir /etc/vomses && \
+	    echo '"cms" "voms-cms-auth.app.cern.ch" "443" "/DC=ch/DC=cern/OU=computers/CN=cms-auth.web.cern.ch" "cms"' > /etc/vomses/voms-cms-auth.app.cern.ch.vomses && \
+        yum clean -y all
+
+        yum --enablerepo=extras install -y epel-release && \
+        yum install -y voms-clients-cpp && \
+    	yum clean -y all
+
+        echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
+        echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \
+        echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
+        localedef -c -i en_US -f UTF-8 en_US.UTF-8
+
+        groupadd -g 1000 cmsusr && adduser -u 1000 -g 1000 -G root cmsusr && \
+        echo "cmsusr ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers && \
+        groupadd -g 1001 cmsinst && adduser -u 1001 -g 1001 cmsinst && \
+        install -d /opt && install -d -o cmsinst /opt/cms
+
+        mkdir -p /cvmfs /afs /eos /pool /code /tmp && \
+        chmod 1777 /afs /eos /pool /code /tmp && \
+        chown -R cmsusr:cmsusr /code
+
+        chmod 755 /opt/cms/entrypoint.sh && \
+        chown -R cmsusr /home/cmsusr && \
+        chmod 755 /home/cmsusr
+
+%runscript
+    exec /opt/cms/entrypoint.sh "$@"
+
+%startscript
+    exec /opt/cms/entrypoint.sh "$@"
diff --git a/apptainer-images/distributions/al9/spec.sin b/apptainer-images/distributions/al9/spec.sin
@@ -0,0 +1,87 @@
+BootStrap: docker
+From: gitlab-registry.cern.ch/linuxsupport/alma9-base:latest
+
+%labels
+    maintainer "CMS Data Preservation and Open Access Group <[email protected]>"
+    org.label-schema.build-date $BUILD_DATE
+    org.label-schema.name "AlmaLinux 9 CMS base OS"
+    org.label-schema.description "AlmaLinux 9 OS capable of using/running CMS software release(s)."
+    org.label-schema.url "http://cms-sw.github.io/"
+    org.label-schema.vcs-ref $VCS_REF
+    org.label-schema.vcs-url $VCS_URL
+    org.label-schema.vendor "CERN"
+    org.label-schema.version $VERSION
+    org.label-schema.schema-version "1.0"
+
+
+%files
+    ../../dotfiles/dot-pythonrc.py /etc/pythonrc.py
+    ../../dotfiles/dot-bashrc /etc/profile.d/bashrc.sh
+    ../../entrypoints/entrypoint-nd.sh /opt/cms/entrypoint.sh
+    cvmfsexec-mnt /opt/cms/cvmfsexec-mnt
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat9/cctools/current/bin/work_queue_worker /usr/bin/work_queue_worker
+
+    # parrot from redhat7, it is ok because tht binary is statically linked. 
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/lobster-171-cd5e3e2c-cvmfs-70dfa0d6/bin/parrot_cvmfs_static_run /usr/bin/parrot_run
+
+
+%environment
+    export LC_ALL=en_US.UTF-8
+    export LANG=en_US.UTF-8
+    export USER=cmsusr
+    export HOME=/home/cmsusr
+
+%post
+    dnf install -y bash && \
+    dnf install -y automake bzip2 bzip2-libs bzip2-devel coreutils-single cmake3 e2fsprogs \
+    e2fsprogs-libs perl file file-libs fontconfig freetype gcc-c++ git \
+    glibc glibc-common glibc-langpack-en glibc-locale-source krb5-libs \
+    libaio libcom_err libcom_err-devel libgomp libICE \
+    libSM libX11 libX11-devel libxcrypt libXcursor libXext \
+    libXext-devel libXft libXft-devel libXi libXinerama \
+    libXmu libXpm libXpm-devel libXrandr libXrender \
+    libglvnd-opengl libtirpc mesa-libGL mesa-libGLU mesa-libGLU-devel \
+    java-1.8.0-openjdk-devel libtool m4 make \
+    ncurses ncurses-libs ncurses-devel nspr nss nss-devel nss-util \
+    openssl openssl-devel openssl-libs libtirpc-devel pcre pcre-devel \
+    patch popt popt-devel python3 python3-pip ninja-build readline readline-devel rpm-build \
+    rsync tcl tcsh time tk wget which zlib zsh tcl-devel tk-devel krb5-devel \
+    bc strace tar zip unzip hostname nano libnsl procps-ng environment-modules yum-utils && \
+    dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && \
+    dnf install -y voms-clients-cpp krb5-workstation python3-psutil myproxy apptainer \
+        python3-requests sudo && \
+    dnf update -y ca-certificates && \
+    dnf install -y dnf-plugins-core yum-plugin-versionlock && \
+    ([ "@EXTRA_PACKAGES@" != "" ] && dnf -y install @EXTRA_PACKAGES@ || true) && \
+    yum clean all
+
+    curl https://storage-ci.web.cern.ch/storage-ci/storageci.key | gpg --import && \
+    yum-config-manager --add-repo "https://storage-ci.web.cern.ch/storage-ci/eos/diopside-depend/el-9/x86_64/" && \
+    yum-config-manager --add-repo "https://storage-ci.web.cern.ch/storage-ci/eos/diopside/tag/testing/el-9/x86_64/" && \
+    dnf update -y && \
+    dnf install --nogpgcheck -y eos-client eos-fusex && \
+    yum clean all
+
+    echo "LC_ALL=en_US.UTF-8" >> /etc/environment && \
+    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \
+    echo "LANG=en_US.UTF-8" > /etc/locale.conf && \
+    localedef -c -i en_US -f UTF-8 en_US.UTF-8
+
+    groupadd -g 1000 cmsusr && adduser -u 1000 -g 1000 -G root cmsusr && \
+    echo "cmsusr ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers && \
+    groupadd -g 1001 cmsinst && adduser -u 1001 -g 1001 cmsinst && \
+    install -d /opt && install -d -o cmsinst /opt/cms
+
+    mkdir -p /cvmfs /afs /eos /pool /code /tmp && \
+    chmod 1777 /afs /eos /pool /code /tmp && \
+    chown -R cmsusr:cmsusr /code
+
+    chmod 755 /opt/cms/entrypoint.sh && \
+    chown -R cmsusr /home/cmsusr && \
+    chmod 755 /home/cmsusr
+
+%runscript
+    exec /opt/cms/entrypoint.sh "$@"
+
+%startscript
+    exec /opt/cms/entrypoint.sh "$@"
diff --git a/apptainer-images/distributions/cc7/spec.sin b/apptainer-images/distributions/cc7/spec.sin
@@ -0,0 +1,94 @@
+Bootstrap: docker
+From: gitlab-registry.cern.ch/linuxsupport/cc7-base:latest
+
+#From: gitlab-registry.cern.ch/linuxsupport/cc7-base:20220601-1.x86_64
+
+# https://gitlab.cern.ch/cms-cloud/cmssw-docker/-/tree/master/cc7-cms?ref_type=heads
+
+%labels
+    maintainer "CMS Data Preservation and Open Access Group <[email protected]>"
+    org.label-schema.build-date $BUILD_DATE
+    org.label-schema.name "CC7 CMS base OS"
+    org.label-schema.description "CC7 OS capable of using/running CMS software release(s)."
+    org.label-schema.url "http://cms-sw.github.io/"
+    org.label-schema.vcs-ref $VCS_REF
+    org.label-schema.vcs-url $VCS_URL
+    org.label-schema.vendor "CERN"
+    org.label-schema.version $VERSION
+    org.label-schema.schema-version "1.0"
+
+%post
+    mkdir -p /home/cmsusr
+    yum-config-manager --save --setopt=epel.skip_if_unavailable=true
+
+
+    yum install -y libX11-devel libXext-devel mesa-libGLU-devel \
+        mesa-libGL-devel libSM libXft libXext \
+        pciutils glx-utils mesa-dri-drivers libX11 libXi libXrender \
+        tcsh zsh tcl tk e2fsprogs perl-ExtUtils-Embed compat-libstdc++-33 libXmu e2fsprogs-libs libXpm bc libaio \
+        tar patch krb5-devel perl-Data-Dumper gcc-4.9 libstdc++6 unzip zip perl-libwww-perl libXpm-devel libXft-devel svn cvs \
+        gcc-c++ gdb strace cern-wrappers krb5-workstation wget hostname readline-devel nano bzip2 perl-Switch perl-Storable \
+        perl-Env packages perl-Thread-Queue CERN-CA-certs tk-devel tcl-devel which python-pip perl voms-clients-cpp \
+        java-1.8.0-openjdk java-1.8.0-openjdk-devel popt libXcursor libXrandr libXinerama nspr nss nss-util file file-libs \
+        readline bzip2-libs python-requests-kerberos libgfortran time python2-psutil python3 \
+        HEP_OSlibs_CC7 git \
+        yum-plugin-ovl openssl \
+        glibc-devel.i686 glibc-devel \
+        glibc-headers \
+        sudo nano && \
+        yum clean -y all
+
+    wget http://repository.egi.eu/sw/production/cas/1/current/repo-files/egi-trustanchors.repo && \
+        mv egi-trustanchors.repo /etc/yum.repos.d/ && \
+        wget http://repository.egi.eu/sw/production/cas/1/gpg-key-eugridpma-rpm-3 && \
+        mv gpg-key-eugridpma-rpm-3 /etc/pki/rpm-gpg/ && \
+        wget http://linuxsoft.cern.ch/wlcg/wlcg-centos7.repo && \
+        mv wlcg-centos7.repo /etc/yum.repos.d/ && \
+        wget http://linuxsoft.cern.ch/wlcg/rpm-gpg-key-wlcg && \
+        mv rpm-gpg-key-wlcg /etc/pki/rpm-gpg/ && \
+        yum install -y ca-policy-egi-core wlcg-repo.noarch wlcg-voms-cms && \
+        yum clean -y all
+
+    yum install -y eos-client && yum clean -y all
+
+    groupadd -g 1000 cmsusr && useradd -u 1000 -g 1000 -G root cmsusr && \
+        echo "cmsusr ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers && \
+        groupadd -g 1001 cmsinst && useradd -u 1001 -g 1001 cmsinst && \
+        mkdir -p /opt && mkdir -p /opt/cms && chown cmsinst /opt/cms
+
+    mkdir -p /cms /cvmfs /afs /eos /pool /code /tmp /var/tmp && \
+        chmod 1777 /cms /cvmfs /afs /eos /pool /code /tmp /var/tmp && \
+        chown -R cmsusr:cmsusr /code
+
+%environment
+    export USER=cmsusr
+    export HOME=/home/cmsusr
+
+%files
+    ../../entrypoints/entrypoint-nd-rh7.sh /opt/cms/entrypoint.sh
+
+    ../../dotfiles/dot-pythonrc.py /etc/pythonrc.py
+    ../../dotfiles/dot-bashrc /etc/profile.d/bashrc.sh
+
+    cvmfsexec-mnt /opt/cms/cvmfsexec-mnt
+
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/7.12.0/bin/work_queue_worker /usr/bin/work_queue_worker
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/7.12.0/bin/work_queue_worker /usr/bin/vine_worker
+
+    # /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/7.3.3/bin/work_queue_worker /usr/bin/work_queue_worker
+    # /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/7.1.9/bin/work_queue_worker /usr/bin/work_queue_worker
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/7.1.0/bin/work_queue_worker /usr/bin/work_queue_worker_old
+    /afs/crc.nd.edu/group/ccl/software/x86_64/redhat7/cctools/lobster-171-cd5e3e2c-cvmfs-70dfa0d6/bin/parrot_cvmfs_static_run /usr/bin/parrot_run
+
+    eos.repo /etc/yum.repos.d/eos.repo
+
+%runscript
+    exec /opt/cms/entrypoint.sh "$@"
+
+%startscript
+    exec /opt/cms/entrypoint.sh "$@"
+
+%post
+    chmod 755 /opt/cms/entrypoint.sh
+    chown -R cmsusr /home/cmsusr
+    chmod 755 /home/cmsusr
diff --git a/apptainer-images/dotfiles/dot-bashrc b/apptainer-images/dotfiles/dot-bashrc
@@ -0,0 +1,7 @@
+## .bashrc/.zshrc
+export PYTHONSTARTUP=/etc/pythonrc.py
+export PS1="(\w) "
+export PROMPT='[%*] %B%n@%m %3~%b $ '
+# define aliases
+alias cmsenv='eval `scramv1 runtime -sh`'
+alias cmsrel='scramv1 project CMSSW'
diff --git a/apptainer-images/dotfiles/dot-pythonrc.py b/apptainer-images/dotfiles/dot-pythonrc.py
@@ -0,0 +1,19 @@
+##
+## for tab-completion
+##
+import rlcompleter, readline
+readline.parse_and_bind('tab: complete')
+readline.parse_and_bind( 'set show-all-if-ambiguous On' )
+
+##
+## for history
+##
+import os, atexit
+histfile = os.path.join(os.environ["HOME"], ".python_history")
+try:
+    readline.read_history_file(histfile)
+except IOError:
+    pass
+atexit.register(readline.write_history_file, histfile)
+del os, atexit, histfile
+del readline
diff --git a/apptainer-images/entrypoints/entrypoint-cms.sh b/apptainer-images/entrypoints/entrypoint-cms.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+set  -e
+
+echo "::: Setting up CMS environment\
+ (works only if /cvmfs is mounted on host) ..."
+if [ -f /cvmfs/cms.cern.ch/cmsset_default.sh ]; then
+  source /cvmfs/cms.cern.ch/cmsset_default.sh
+  echo "::: Setting up CMS environment... [done]"
+else
+  echo "::: Could not set up CMS environment... [ERROR]"
+  echo "::: /cvmfs/cms.cern.ch/cmsset_default.sh not found/available"
+fi
+
+exec "$@"
diff --git a/apptainer-images/entrypoints/entrypoint-nd-rh7.sh b/apptainer-images/entrypoints/entrypoint-nd-rh7.sh
@@ -0,0 +1,67 @@
+#! /bin/bash
+
+trap clean_cvmfsexec EXIT
+
+function clean_cvmfsexec ()
+{
+    if [[ -z "${NDCMS_ENTRYPOINT_RECURSIVE}"  &&  -n "${CVMFS_TMP_EXEC_DIR}" ]]
+    then
+        if [[ -e "${CVMFS_TMP_EXEC_DIR}/mnt/afs/crc.nd.edu" ]]
+        then
+            # safety check, do not remove if afs is still mounted!
+            return
+        else
+            rm -rf "${CVMFS_TMP_EXEC_DIR}"
+        fi
+    fi
+}
+
+
+if [[ -z "${NDCMS_ENTRYPOINT_RECURSIVE}" ]]
+then
+    echo "::: Setting up CMS environment\
+        (works only if /cvmfs is mounted on host) ..."
+fi
+
+
+SETUP=/cvmfs/cms.cern.ch/cmsset_default.sh
+if [ -f "${SETUP}" ]
+then
+    source "${SETUP}"
+    source /cvmfs/oasis.opensciencegrid.org/osg-software/osg-wn-client/3.6/current/el7-x86_64/setup.sh
+    echo "::: Setting up CMS environment... [done]"
+
+    if [[ -n "${CVMFS_TMP_EXEC_DIR}" ]]
+    then
+        touch ${CVMFS_TMP_EXEC_DIR}/success
+    fi
+    exec "$@"
+elif [ -n "${NDCMS_ENTRYPOINT_RECURSIVE}" ];
+then
+    echo "::: /cvmfs/cms.cern.ch/cmsset_default.sh not found/available"
+    echo "::: Could not set up CMS environment... [ERROR]"
+else
+    REPOSITORIES="cms.cern.ch oasis.opensciencegrid.org"
+    if [ -n "${CVMFS_REPOSITORIES}" ];
+    then
+        REPOSITORIES="${REPOSITORIES} $(echo ${CVMFS_REPOSITORIES} | sed 's/,/ /')"
+    fi
+
+    export NDCMS_ENTRYPOINT_RECURSIVE=yes
+    export CVMFS_TMP_EXEC_DIR=$(mktemp -d -t cvmfsexec-XXXXXX)
+
+    cp /opt/cms/cvmfsexec-mnt ${CVMFS_TMP_EXEC_DIR}
+    ${CVMFS_TMP_EXEC_DIR}/cvmfsexec-mnt ${REPOSITORIES} -- "${BASH_SOURCE[0]}" "$@"
+    status=$?
+
+    # if exec failed, unset so that the cleanup knows the temp dir is not needed anymore
+    unset NDCMS_ENTRYPOINT_RECURSIVE
+
+    # fall back only if cvmfsexec failed to set env in recursive call
+    if [[ -f ${CVMFS_TMP_EXEC_DIR}/success ]]
+    then
+        exit ${status}
+    else
+        exec "$@"
+    fi
+fi