Merge Release 2.4.1

Merge Release 2.4.1

Author: demartinofra

CHANGELOG.md

@@ -3,6 +3,43 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
+2.4.1
+-----
+
+**ENHANCEMENTS**
+- Install IntelMPI on Alinux, Centos 7 and Ubuntu 1604
+- Upgrade EFA to version 1.4.1
+- Run all node daemons and cookbook recipes in isolated Python virtualenvs. This allows our code to always run with the
+  required Python dependencies and solves all conflicts and runtime failures that were being caused by user packages
+  installed in the system Python
+
+**CHANGES**
+- Torque: upgrade to version 6.1.2
+- Run all node daemons with Python 3.6
+- Torque: changed following parameters in global configuration:
+  - `server node_check_rate = 120` - Specifies the minimum duration (in seconds)
+    that a node can fail to send a status update before being marked down by the
+    pbs_server daemon. Previously was 600. This reduces scaling reaction times in
+    case of instance failure or unexpected termination (especially with spot)
+  - `server node_ping_rate = 60` - Specifies the maximum interval (in seconds)
+    between successive "pings" sent from the pbs_server daemon to the pbs_mom
+    daemon to determine node/daemon health. Previously was 300. Setting it to half
+    the node_check_rate.
+  - `server timeout_for_job_delete = 30` - The specific timeout used when deleting
+    jobs because the node they are executing on is being deleted. Previously was
+    120. This prevents job deletion to hang for more than 30 seconds when the node
+    they are running on is being deleted.
+  - `server timeout_for_job_requeue = 30` - The specific timeout used when requeuing
+    jobs because the node they are executing on is being deleted. Previously was
+    120. This prevents node deletion to hang for more than 30 seconds when a job
+    cannot be rescheduled.
+
+**BUG FIXES**
+- Restore correct value for `filehandle_limit` that was getting reset when setting `memory_limit` for EFA
+- Torque: fix configuration of server operators that was preventing compute nodes from disabling themselves
+  before termination
+
+
 2.4.0
 -----
 

amis/build_ami.sh

@@ -20,14 +20,14 @@
 #   build-date: timestamp to append to the AMIs names (optional)
 
 requirements_check() {
-    which packer >/dev/null 2>&1
+    packer build --help >/dev/null 2>&1
     if [ $? -ne 0 ] ; then
       echo "packer command not found. Is Packer installed?"
       echo "Please visit https://www.packer.io/downloads.html for instruction on how to download and install"
       exit 1
     fi
 
-    which berks >/dev/null 2>&1
+    berks vendor --help >/dev/null 2>&1
     if [ $? -ne 0 ] ; then
       echo "berks command not found. Is ChefDK installed?"
       echo "Please visit https://downloads.chef.io/chefdk/ for instruction on how to download and install"
@@ -160,6 +160,9 @@ do_command() {
       export BUILD_DATE=${_build_date}
     fi
 
+    # set it to try for 1 hour, this is to resolve ami copy timeout issue
+    # https://github.com/hashicorp/packer/issues/6536
+    export AWS_TIMEOUT_SECONDS=3600
 
     case ${_os} in
       all)
@@ -190,4 +193,4 @@ main() {
     do_command
 }
 
-main "$@"
+main "$@"

amis/packer_variables.json

@@ -1,6 +1,6 @@
 {
-  "parallelcluster_version": "2.4.0",
-  "parallelcluster_cookbook_version": "2.4.0",
+  "parallelcluster_version": "2.4.1",
+  "parallelcluster_cookbook_version": "2.4.1",
   "chef_version": "14.2.0",
   "ridley_version": "5.1.1",
   "berkshelf_version": "7.0.4"

attributes/default.rb

@@ -18,17 +18,30 @@
 default['cfncluster']['sources_dir'] = "#{node['cfncluster']['base_dir']}/sources"
 default['cfncluster']['scripts_dir'] = "#{node['cfncluster']['base_dir']}/scripts"
 default['cfncluster']['license_dir'] = "#{node['cfncluster']['base_dir']}/licenses"
+# Python Version
+default['cfncluster']['python-version'] = '3.6.9'
+# Virtualenv Cookbook Name
+default['cfncluster']['cookbook_virtualenv'] = 'cookbook_virtualenv'
+# Virtualenv Node Name
+default['cfncluster']['node_virtualenv'] = 'node_virtualenv'
+# Cookbook Virtualenv Path
+default['cfncluster']['cookbook_virtualenv_path'] = "/root/.pyenv/versions/#{node['cfncluster']['python-version']}/envs/#{node['cfncluster']['cookbook_virtualenv']}"
+# Node Virtualenv Path
+default['cfncluster']['node_virtualenv_path'] = "/root/.pyenv/versions/#{node['cfncluster']['python-version']}/envs/#{node['cfncluster']['node_virtualenv']}"
+# Intel MPI
+default['cfncluster']['intelmpi']['url'] = "http://registrationcenter-download.intel.com/akdlm/irc_nas/tec/15553/aws_impi.sh"
+default['cfncluster']['intelmpi']['version'] = '2019.4.243'
+default['cfncluster']['intelmpi']['modulefile'] = "/opt/intel/impi/#{node['cfncluster']['intelmpi']['version']}/intel64/modulefiles/mpi"
 # Python packages
-default['cfncluster']['cfncluster-version'] = '2.4.0'
-default['cfncluster']['cfncluster-node-version'] = '2.4.0'
-default['cfncluster']['supervisor-version'] = '3.4.0'
+default['cfncluster']['cfncluster-version'] = '2.4.1'
+default['cfncluster']['cfncluster-node-version'] = '2.4.1'
 # URLs to software packages used during install recipes
 # Gridengine software
 default['cfncluster']['sge']['version'] = '8.1.9'
 default['cfncluster']['sge']['url'] = 'https://arc.liv.ac.uk/downloads/SGE/releases/8.1.9/sge-8.1.9.tar.gz'
 # Torque software
-default['cfncluster']['torque']['version'] = '6.0.2'
-default['cfncluster']['torque']['url'] = 'https://github.com/adaptivecomputing/torque/archive/6.0.2.tar.gz'
+default['cfncluster']['torque']['version'] = '6.1.2'
+default['cfncluster']['torque']['url'] = 'https://github.com/adaptivecomputing/torque/archive/6.1.2.tar.gz'
 # Slurm software
 default['cfncluster']['slurm']['version'] = '18-08-6-2'
 default['cfncluster']['slurm']['url'] = 'https://github.com/SchedMD/slurm/archive/slurm-18-08-6-2.tar.gz'
@@ -62,20 +75,27 @@
 default['openssh']['server']['subsystem'] = 'sftp /usr/libexec/openssh/sftp-server'
 default['openssh']['client']['gssapi_authentication'] = 'yes'
 
+# ulimit settings
+default['cfncluster']['filehandle_limit'] = 10000
+default['cfncluster']['memory_limit'] = 'unlimited'
+
 # Platform defaults
 case node['platform_family']
 when 'rhel', 'amazon'
 
   default['cfncluster']['kernel_devel_pkg']['name'] = "kernel-devel"
   default['cfncluster']['kernel_devel_pkg']['version'] = node['kernel']['release'].chomp('.x86_64')
 
+  # Modulefile Directory
+  default['cfncluster']['modulefile_dir'] = "/usr/share/Modules/modulefiles"
+
   case node['platform']
   when 'centos', 'redhat', 'scientific' # ~FC024
     default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
                                                 libXmu-devel hwloc-devel db4-devel tcl-devel automake autoconf pyparted libtool
                                                 httpd boost-devel redhat-lsb mlocate mpich-devel openmpi-devel R atlas-devel
                                                 blas-devel fftw-devel libffi-devel openssl-devel dkms mysql-devel libedit-devel
-                                                libical-devel postgresql-devel postgresql-server sendmail mdadm]
+                                                libical-devel postgresql-devel postgresql-server sendmail mdadm python python-pip]
 
     # Lustre Drivers for Centos 6
     default['cfncluster']['lustre']['version'] = '2.10.6'
@@ -87,7 +107,7 @@
                                                   libXmu-devel hwloc-devel libdb-devel tcl-devel automake autoconf pyparted libtool
                                                   httpd boost-devel redhat-lsb mlocate lvm2 mpich-devel R atlas-devel
                                                   blas-devel fftw-devel libffi-devel openssl-devel dkms mariadb-devel libedit-devel
-                                                  libical-devel postgresql-devel postgresql-server sendmail libxml2-devel libglvnd-devel mdadm]
+                                                  libical-devel postgresql-devel postgresql-server sendmail libxml2-devel libglvnd-devel mdadm python python-pip]
       if node['platform_version'].split('.')[1] == '6'
         # Lustre Drivers for Centos 7.6
         default['cfncluster']['lustre']['version'] = '2.10.6'
@@ -108,7 +128,7 @@
     default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh openssl-devel ncurses-devel pam-devel net-tools openmotif-devel
                                                 libXmu-devel hwloc-devel db4-devel tcl-devel automake autoconf pyparted libtool
                                                 httpd boost-devel redhat-lsb mlocate mpich-devel R atlas-devel fftw-devel
-                                                libffi-devel openssl-devel dkms mysql-devel libedit-devel postgresql-devel postgresql-server
+                                                libffi-devel dkms mysql-devel libedit-devel postgresql-devel postgresql-server
                                                 sendmail cmake byacc libglvnd-devel mdadm]
   end
 
@@ -125,11 +145,13 @@
   default['cfncluster']['base_packages'] = %w[vim ksh tcsh zsh libssl-dev ncurses-dev libpam-dev net-tools libhwloc-dev dkms
                                               tcl-dev automake autoconf python-parted libtool librrd-dev libapr1-dev libconfuse-dev
                                               apache2 libboost-dev libdb-dev tcsh libssl-dev libncurses5-dev libpam0g-dev libxt-dev
-                                              libmotif-dev libxmu-dev libxft-dev libhwloc-dev man-db lvm2 libmpich-dev
+                                              libmotif-dev libxmu-dev libxft-dev libhwloc-dev man-db lvm2 libmpich-dev python python-pip
                                               r-base libatlas-dev libblas-dev libfftw3-dev libffi-dev libssl-dev libxml2-dev mdadm]
   if node['platform_version'] == '14.04'
     default['cfncluster']['base_packages'].push('libopenmpi-dev')
   end
+  # Modulefile Directory
+  default['cfncluster']['modulefile_dir'] = "/usr/share/modules/modulefiles"
   default['cfncluster']['kernel_generic_pkg'] = "linux-generic"
   default['cfncluster']['kernel_extra_pkg'] = "linux-image-extra-#{node['kernel']['release']}"
   default['cfncluster']['ganglia']['apache_user'] = 'www-data'

files/default/attachVolume.py

@@ -1,12 +1,10 @@
-#!/usr/bin/env python
-
 import sys
-import parted
+import subprocess
 import os
-import urllib2
+import requests
 import boto3
 import time
-import ConfigParser
+import configparser
 from botocore.config import Config
 
 
@@ -23,24 +21,36 @@ def convert_dev(dev):
     else:
         return dev
 
+def get_all_devices():
+    # lsblk -d -n
+    # xvda 202:0    0  17G  0 disk
+    # xvdb 202:16   0  20G  0 disk /shared
+    command = ["/bin/lsblk", "-d", "-n"]
+
+    try:
+        output = subprocess.check_output(command, stderr=subprocess.STDOUT, universal_newlines=True).split("\n")
+        return ["/dev/{}".format(line.split()[0]) for line in output if len(line.split()) > 0]
+    except subprocess.CalledProcessError as e:
+        print("Failed to get devices with lsblk -d -n")
+        raise e
 
 def main():
     # Get EBS volume Id
     try:
         volumeId = str(sys.argv[1])
     except IndexError:
-        print "Provide an EBS volume ID to attach i.e. vol-cc789ea5"
+        print("Provide an EBS volume ID to attach i.e. vol-cc789ea5")
         sys.exit(1)
 
     # Get instance ID
-    instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
+    instanceId = requests.get("http://169.254.169.254/latest/meta-data/instance-id").text
 
     # Get region
-    region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
+    region = requests.get("http://169.254.169.254/latest/meta-data/placement/availability-zone").text
     region = region[:-1]
 
     # Generate a list of system paths minus the root path
-    paths = [convert_dev(device.path) for device in parted.getAllDevices()]
+    paths = [convert_dev(device) for device in get_all_devices()]
 
     # List of possible block devices
     blockDevices = ['/dev/sdb', '/dev/sdc', '/dev/sdd', '/dev/sde', '/dev/sdf', '/dev/sdg', '/dev/sdh',
@@ -52,7 +62,7 @@ def main():
     availableDevices = [a for a in blockDevices if a not in paths]
 
     # Parse configuration file to read proxy settings
-    config = ConfigParser.RawConfigParser()
+    config = configparser.RawConfigParser()
     config.read('/etc/boto.cfg')
     proxy_config = Config()
     if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
@@ -72,12 +82,12 @@ def main():
     x = 0
     while state != "attached":
         if x == 36:
-            print "Volume %s failed to mount in 180 seconds." % volumeId
+            print("Volume %s failed to mount in 180 seconds." % volumeId)
             exit(1)
         if state in ["busy" or "detached"]:
-            print "Volume %s in bad state %s" % (volumeId, state)
+            print("Volume %s in bad state %s" % (volumeId, state))
             exit(1)
-        print "Volume %s in state %s ... waiting to be 'attached'" % (volumeId, state)
+        print("Volume %s in state %s ... waiting to be 'attached'" % (volumeId, state))
         time.sleep(5)
         x += 1
         try:

files/default/compute_ready

@@ -1,12 +1,10 @@
-#!/usr/bin/env python
-
-import urllib2
+import requests
 import sys
 import os
 import syslog
 import time
 import boto3
-import ConfigParser
+import configparser
 from botocore.config import Config
 
 
@@ -31,14 +29,14 @@ def main():
         dev = '/dev/' + dev
 
     # Get instance ID
-    instanceId = urllib2.urlopen("http://169.254.169.254/latest/meta-data/instance-id").read()
+    instanceId = requests.get("http://169.254.169.254/latest/meta-data/instance-id").text
 
     # Get region
-    region = urllib2.urlopen("http://169.254.169.254/latest/meta-data/placement/availability-zone").read()
+    region = requests.get("http://169.254.169.254/latest/meta-data/placement/availability-zone").text
     region = region[:-1]
 
     # Parse configuration file to read proxy settings
-    config = ConfigParser.RawConfigParser()
+    config = configparser.RawConfigParser()
     config.read('/etc/boto.cfg')
     proxy_config = Config()
     if config.has_option('Boto', 'proxy') and config.has_option('Boto', 'proxy_port'):
@@ -53,7 +51,7 @@ def main():
     devices = ec2.describe_instance_attribute(InstanceId=instanceId, Attribute='blockDeviceMapping').get('BlockDeviceMappings')
     devmap = dict((d.get('DeviceName'), d) for d in devices)
     x = 0
-    while not devmap.has_key(dev):
+    while dev not in devmap:
         if x == 36:
             syslog.syslog("Dev %s did not appears in 180 seconds." % dev)
             sys.exit(1)

files/default/ec2-volid.rules

@@ -0,0 +1,4 @@
+awscli
+boto3
+supervisor
+requests

files/default/ec2_dev_2_volid.py

@@ -0,0 +1 @@
+set path = (/opt/torque/bin /opt/torque/sbin $path)