Release 2.5.0

Merge Release 2.5.0

Author: Sean Smith

.flake8

@@ -0,0 +1,25 @@
+[flake8]
+ignore =
+    D100, # Missing docstring in public module
+    D104, # Missing docstring in public package
+    D107, # Missing docstring in __init__
+    D103, # Missing docstring in public function
+    W503, # line break before binary operator => Conflicts with black style.
+    D413, # Missing blank line after last section
+exclude =
+    .tox,
+    .git,
+    .pytest_cache,
+    docs/source/conf.py,
+    build,
+    dist,
+    tests/fixtures/*,
+    *.pyc,
+    *.egg-info,
+    .cache,
+    .eggs
+max-complexity = 10
+max-line-length = 120
+import-order-style = google
+application-import-names = flake8
+format = ${cyan}%(path)s${reset}:${yellow_bold}%(row)d${reset}:${green_bold}%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

.gitignore

@@ -18,3 +18,5 @@ bin/*
 
 build.log
 build-*.log
+
+.tox/

.isort.cfg

@@ -0,0 +1,13 @@
+[settings]
+line_length=120
+known_future_library=future
+known_third_party=boto3,botocore,awscli,tabulate,argparse,configparser,pytest,pytest,pytest-datadir,pytest-html,pytest-rerunfailures,pytest-xdist,argparse,retrying,junitparser,Jinja2
+# 3 - Vertical Hanging Indent
+# from third_party import (
+#     lib1,
+#     lib2,
+#     lib3,
+#     lib4,
+# )
+multi_line_output=3
+include_trailing_comma=true

.kitchen.cloud.yml

@@ -35,7 +35,7 @@ platforms:
       block_device_mappings:
         - device_name: /dev/xvda
           ebs:
-            volume_size: <%= ENV['VOLUME_SIZE'] || 17 %>
+            volume_size: <%= ENV['VOLUME_SIZE'] || 25 %>
             volume_type: gp2
             delete_on_termination: true
         - device_name: /dev/xvdba
@@ -96,7 +96,7 @@ platforms:
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
-            volume_size: <%= ENV['VOLUME_SIZE'] || 17 %>
+            volume_size: <%= ENV['VOLUME_SIZE'] || 25 %>
             volume_type: gp2
             delete_on_termination: true
         - device_name: /dev/xvdba
@@ -157,7 +157,7 @@ platforms:
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
-            volume_size: <%= ENV['VOLUME_SIZE'] || 17 %>
+            volume_size: <%= ENV['VOLUME_SIZE'] || 25 %>
             volume_type: gp2
             delete_on_termination: true
         - device_name: /dev/xvdba
@@ -211,14 +211,14 @@ platforms:
     transport:
       username: centos
       ssh_key: <%= ENV['EC2_SSH_KEY_PATH'] %>
-  - name: ubuntu-14-04-lts
+  - name: ubuntu-16-04-lts
     driver_plugin: ec2
     driver_config:
-      image_id: <%= ENV['UBUNTU1404_IMAGE_ID'] || "ami-c29e1cb8" %>
+      image_id: <%= ENV['UBUNTU1604_IMAGE_ID'] || "ami-aa2ea6d0" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
-            volume_size: <%= ENV['VOLUME_SIZE'] || 17 %>
+            volume_size: <%= ENV['VOLUME_SIZE'] || 25 %>
             volume_type: gp2
             delete_on_termination: true
         - device_name: /dev/xvdba
@@ -272,14 +272,14 @@ platforms:
     transport:
       username: ubuntu
       ssh_key: <%= ENV['EC2_SSH_KEY_PATH'] %>
-  - name: ubuntu-16-04-lts
+  - name: ubuntu-18-04-lts
     driver_plugin: ec2
     driver_config:
-      image_id: <%= ENV['UBUNTU1604_IMAGE_ID'] || "ami-aa2ea6d0" %>
+      image_id: <%= ENV['UBUNTU1804_IMAGE_ID'] || "ami-05ecb1463f8f1ee4b" %>
       block_device_mappings:
         - device_name: /dev/sda1
           ebs:
-            volume_size: <%= ENV['VOLUME_SIZE'] || 17 %>
+            volume_size: <%= ENV['VOLUME_SIZE'] || 25 %>
             volume_type: gp2
             delete_on_termination: true
         - device_name: /dev/xvdba

.travis.yml

@@ -1,8 +1,36 @@
-language: ruby
-rvm:
-  - 2.4.0
-before_install:
-  - wget https://releases.hashicorp.com/vagrant/2.0.0/vagrant_2.0.0_x86_64.deb
-  - sudo dpkg -i vagrant_2.0.0_x86_64.deb
-install: bundle install --without aws
-script: bundle exec rake
+matrix:
+  include:
+    - language: ruby
+      rvm:
+        - 2.4.0
+      before_install:
+        - wget https://releases.hashicorp.com/vagrant/2.0.0/vagrant_2.0.0_x86_64.deb
+        - sudo dpkg -i vagrant_2.0.0_x86_64.deb
+      install: bundle install --without aws
+      script: bundle exec rake
+
+    - language: python
+      sudo: required
+      dist: xenial
+      python: "3.6"
+      install:
+        - pip install tox-travis
+      script: tox
+
+    - language: python
+      sudo: required
+      dist: xenial
+      python: "3.7"
+      install:
+        - pip install tox-travis
+      script: tox
+
+    - language: python
+      sudo: required
+      dist: xenial
+      python: "3.7"
+      name: Python Code Checks
+      env: TOXENV=code-linters
+      install:
+        - pip install tox-travis
+      script: tox

Berksfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://supermarket.chef.io'
 
 metadata

CHANGELOG.md

@@ -3,6 +3,43 @@ aws-parallelcluster-cookbook CHANGELOG
 
 This file is used to list changes made in each version of the AWS ParallelCluster cookbook.
 
+2.5.0
+-----
+
+**ENHANCEMENTS**
+- Install NICE DCV on Centos 7 (this includes Gnome and Xorg packages).
+- Install Intel Parallel Studio 2019.5 Runtime in Centos 7 AMI and share /opt/intel over NFS.
+- Add support for Ubuntu 18.
+
+**CHANGES**
+- Remove support for Ubuntu 14.
+- Upgrade Intel MPI to version U5.
+- Upgrade EFA Installer to version 1.6.2, this also upgrades Open MPI to 4.0.2.
+- Upgrade NVIDIA driver to Tesla version 418.87.
+- Upgrade CUDA library to version 10.1.
+- Upgrade Slurm to version 19.05.3-2.
+- Slurm: changed following parameters in global configuration:
+  - `SelectType=cons_tres`, `SelectTypeParameter=CR_CPU_Memory`, `GresTypes=gpu`: needed to enable support for GPU
+    scheduling.
+  - `EnforcePartLimits=ALL`: jobs which exceed a partition's size and/or time limits will be rejected at submission
+    time.
+  - Removed `FastSchedule` since deprecated.
+  - `SlurmdTimeout=180`, `UnkillableStepTimeout=180`: to allow scheduler to recover especially when under heavy load.
+- Echo compute instance type and memory information in COMPUTE_READY message
+- Changes to sshd config:
+  - Disable X11Forwarding by default
+  - Limit SSH Ciphers to 
+    `aes128-cbc,aes192-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr,[email protected],[email protected]`
+  - Limit SSH MACs to `[email protected],[email protected],hmac-sha2-512,hmac-sha2-256`
+- Increase default root volume to 25GB.
+- Enable `flock user_xattr noatime` Lustre options by default everywhere and
+  `x-systemd.automount x-systemd.requires=lnet.service` for systemd based systems.
+- Install EFA in China AMIs. 
+
+**BUG FIXES**
+- Fix Ganglia not starting on Ubuntu 16
+- Fix bug that was preventing nodes to mount partitioned EBS volumes.
+
 2.4.1
 -----
 

Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gem 'berkshelf'

Rakefile

@@ -1,4 +1,5 @@
 #!/usr/bin/env rake
+# frozen_string_literal: true
 
 # chefspec task against spec/*_spec.rb
 require 'rspec/core/rake_task'

amis/build_ami.sh

@@ -12,7 +12,7 @@
 # AMI IDs from this region. Moreover, the CentOs AMIs are private to the AWS ParallelCluster account.
 #
 # Usage: build_ami.sh --os <os> --region <region> --partition <partition> [--public] [--custom] [--build-date <build-date>]
-#   os: the os to build (supported values: all|centos6|centos7|alinux|ubuntu1404|ubuntu1604)
+#   os: the os to build (supported values: all|centos6|centos7|alinux|ubuntu1604)
 #   partition: partition to build in (supported values: commercial|govcloud|china)
 #   region: region to copy ami too (supported values: all|us-east-1|us-gov-west-1|...)
 #   custom: specifies to create the AMI from a custom AMI-id, which must be specified by variable CUSTOM_AMI_ID in the environment (optional)
@@ -101,7 +101,7 @@ parse_options() {
 check_options() {
     set -e
 
-    available_os="centos6 centos7 alinux ubuntu1404 ubuntu1604"
+    available_os="centos6 centos7 alinux ubuntu1604 ubuntu1804"
     cwd="$(dirname $0)"
     tmp_dir=$(mktemp -d)
     export VENDOR_PATH="${tmp_dir}/vendor/cookbooks"
@@ -171,7 +171,7 @@ do_command() {
           RC=$?
         done
         ;;
-      centos6|centos7|alinux|ubuntu1404|ubuntu1604)
+      centos6|centos7|alinux|ubuntu1604|ubuntu1804)
         packer build -color=false -var-file="${cwd}/packer_variables.json" -only=${only} "${cwd}/packer_${_os}.json"
         RC=$?
         ;;