Merge pull request #482 from awslabs/develop

Release 1.5.3

Author: sean-smith

CHANGELOG.rst

@@ -2,6 +2,10 @@
 CHANGELOG
 =========
 
+1.5.3
+=====
+* Add support for GovCloud, us-gov-west-1 region
+
 1.5.2
 =====
 * feature:``cfncluster``: Added ClusterUser as a stack output. This makes it easier to get the username of the head node.

amis.txt

@@ -13,6 +13,7 @@ eu-west-3: ami-0b1472bf5fed029bd
 sa-east-1: ami-0a390e1998fb6b22f
 us-east-1: ami-0fd6854902340b407
 us-east-2: ami-04fc90a3f60cb3dc7
+us-gov-west-1: ami-af2cbece
 us-west-1: ami-0054acc8361046ccc
 us-west-2: ami-01382f63e2667a7d1
 # centos6
@@ -64,6 +65,7 @@ eu-west-3: ami-0e8f93d19635c5d49
 sa-east-1: ami-048422e42509a1596
 us-east-1: ami-0ba2fcbf9fa378762
 us-east-2: ami-05aea9ea89d577194
+us-gov-west-1: ami-4aca572b
 us-west-1: ami-0089ace30f2612d13
 us-west-2: ami-06142a55749b7912b
 # ubuntu1604
@@ -81,5 +83,6 @@ eu-west-3: ami-09c4322a65746f77b
 sa-east-1: ami-0740b75b4caaa3b77
 us-east-1: ami-05790a6a634720a0d
 us-east-2: ami-0a0fcd1219e40a8f5
+us-gov-west-1: ami-3dc8555c
 us-west-1: ami-056472d51cf2fed45
 us-west-2: ami-0702d59e57ada76b8

cli/cfncluster/cfnconfig.py

@@ -177,7 +177,7 @@ def __init__(self, args):
                                                      'URL', self.template_url)
                 except configparser.NoOptionError:
                     if self.region == 'us-gov-west-1':
-                        self.template_url = ('https://s3-%s.amazonaws.com/cfncluster-%s/templates/cfncluster-%s.cfn.json'
+                        self.template_url = ('https://s3-%s.amazonaws.com/%s-cfncluster/templates/cfncluster-%s.cfn.json'
                                              % (self.region, self.region, self.version))
                     else:
                         self.template_url = ('https://s3.amazonaws.com/%s-cfncluster/templates/cfncluster-%s.cfn.json'

cli/cfncluster/easyconfig.py

@@ -25,7 +25,7 @@
 from . import cfnconfig
 
 logger = logging.getLogger('cfncluster.cfncluster')
-unsupported_regions = ['ap-northeast-3']
+unsupported_regions = ['ap-northeast-3', 'cn-north-1', 'cn-northwest-1']
 
 def prompt(prompt, default_value=None, hidden=False, options=None):
     if hidden and default_value is not None:

cli/setup.py

@@ -20,7 +20,7 @@ def read(fname):
     return open(os.path.join(os.path.dirname(__file__), fname)).read()
 
 console_scripts = ['cfncluster = cfncluster.cli:main']
-version = "1.5.2"
+version = "1.5.3"
 requires = ['boto3>=1.7.33', 'awscli>=1.11.175', 'future>=0.16.0']
 
 if sys.version_info[:2] == (2, 6):

cloudformation/cfncluster.cfn.json

@@ -1741,6 +1741,11 @@
         "centos7": "ami-0df85f5bc2d980e57",
         "ubuntu1404": "ami-06142a55749b7912b",
         "ubuntu1604": "ami-0702d59e57ada76b8"
+      },
+      "us-gov-west-1": {
+        "alinux": "ami-af2cbece",
+        "ubuntu1404": "ami-4aca572b",
+        "ubuntu1604": "ami-3dc8555c"
       }
     },
     "OSFeatures": {
@@ -1767,7 +1772,7 @@
     },
     "CfnClusterVersions": {
       "default": {
-        "cfncluster": "cfncluster-1.5.2",
+        "cfncluster": "cfncluster-1.5.3",
         "cookbook": "cfncluster-cookbook-1.5.2",
         "chef": "14.2.0",
         "ridley": "5.1.1",

docs/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.5'
 # The full version, including alpha/beta/rc tags.
-release = '1.5.2'
+release = '1.5.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/iam.rst

@@ -8,7 +8,7 @@ IAM in CfnCluster
 
 
 CfnCluster utilizes multiple AWS services to deploy and operate a cluster. The services used are listed in the :ref:`AWS Services used in CfnCluster <aws_services>` section of the documentation.
- 
+
 CfnCluster uses EC2 IAM roles to enable instances access to AWS services for the deployment and operation of the cluster. By default the EC2 IAM role is created as part of the cluster creation by CloudFormation. This means that the user creating the cluster must have the appropriate level of permissions
 
 Defaults
@@ -176,7 +176,9 @@ CfnClusterUserPolicy
                   "ec2:DeleteSecurityGroup",
                   "ec2:DisassociateAddress",
                   "ec2:RevokeSecurityGroupIngress",
-                  "ec2:ReleaseAddress"
+                  "ec2:ReleaseAddress",
+                  "ec2:CreatePlacementGroup",
+                  "ec2:DeletePlacementGroup"
               ],
               "Effect": "Allow",
               "Resource": "*"
@@ -308,7 +310,9 @@ CfnClusterUserPolicy
               "Action": [
                   "iam:PassRole",
                   "iam:CreateRole",
-                  "iam:DeleteRole"
+                  "iam:DeleteRole",
+                  "iam:GetRole",
+                  "iam:SimulatePrincipalPolicy"
               ],
               "Effect": "Allow",
               "Resource": "arn:aws:iam::<AWS ACCOUNT ID>:role/<CFNCLUSTER EC2 ROLE NAME>"

tests/cluster-check.sh

@@ -43,6 +43,19 @@ sge_get_slots() {
     echo ${ppn}
 }
 
+slurm_get_slots() {
+    local -- ppn i=0
+    ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p')
+    # wait 15 secs before giving up retrieving the slots per host
+    while [ -z "${ppn}" -a $i -lt 15 ]; do
+        sleep 1
+        i=$((i+1))
+        ppn=$(scontrol show nodes -o | head -n 1 | sed -n -e 's/^.* CPUTot=\([0-9]\+\) .*$/\1/p')
+    done
+
+    echo ${ppn}
+}
+
 torque_get_slots() {
     local -- chost ppn i=0
 
@@ -76,6 +89,12 @@ set -e
 # less than 8 minutes in order for the test to succeed.
 
 if test "$scheduler" = "slurm" ; then
+    _ppn=$(slurm_get_slots)
+    if [ -z "${_ppn}" ]; then
+        >&2 echo "The number of slots per instance couldn't be retrieved, no compute nodes available in Slurm cluster"
+        exit 1
+    fi
+
     cat > job1.sh <<EOF
 #!/bin/bash
 srun sleep ${_sleepjob1}
@@ -90,8 +109,8 @@ EOF
     chmod +x job1.sh job2.sh
     rm -f job1.done job2.done
 
-    sbatch -N 1 ./job1.sh
-    sbatch -N 1 ./job2.sh
+    sbatch -N 1 -n ${_ppn} ./job1.sh
+    sbatch -N 1 -n ${_ppn} ./job2.sh
 
 elif test "$scheduler" = "sge" ; then
     # get the slots per node count of the first real node (one with a

util/generate-ami-list.py

@@ -25,18 +25,16 @@
 import sys
 from collections import OrderedDict
 
-
-owners = ["247102896272"]
 distros = OrderedDict([("alinux", "amzn"), ("centos6", "centos6"), ("centos7", "centos7"), ("ubuntu1404", "ubuntu-1404"), ("ubuntu1604", "ubuntu-1604")])
 
 
-def get_ami_list(regions, date, version):
+def get_ami_list(regions, date, version, owner):
     amis_json = {}
 
     for region_name in regions:
         try:
             ec2 = boto3.client('ec2', region_name=region_name)
-            images = ec2.describe_images(Owners=owners, Filters=[{'Name': 'name', "Values": ["cfncluster-%s*%s" % (version, date)]}])
+            images = ec2.describe_images(Owners=[owner], Filters=[{'Name': 'name', "Values": ["cfncluster-%s*%s" % (version, date)]}])
 
             amis = OrderedDict()
             for image in images.get('Images'):
@@ -76,20 +74,32 @@ def convert_json_to_txt(regions, amis_json):
     parser.add_argument('--date', type=str, help='release date [timestamp] (e.g. 201801112350)', required=True)
     parser.add_argument('--json-file', type=str, help='json output file path', required=False, default="amis.json")
     parser.add_argument('--txt-file', type=str, help='txt output file path', required=False, default="amis.txt")
+    parser.add_argument('--account-id', type=str, help='account id that owns the amis', required=False,  default="247102896272")
+    parser.add_argument('--append', type=str, help='append new amis to current amis.txt', required=False, default=False)
+    parser.add_argument('--cloudformation-template', type=str, help='path to cloudfomation template', required=False, default='cloudformation/cfncluster.cfn.json')
     args = parser.parse_args()
 
     # get all regions
     ec2 = boto3.client('ec2')
     regions = sorted(r.get('RegionName') for r in ec2.describe_regions().get('Regions'))
 
     # get ami list
-    amis_json = get_ami_list(regions=regions, date=args.date, version=args.version)
+    amis_json = get_ami_list(regions=regions, date=args.date, version=args.version, owner=args.account_id)
 
     # write amis.json file
     amis_json_file = open(args.json_file, "w")
     json.dump(amis_json, amis_json_file, indent=2, sort_keys=True)
     amis_json_file.close()
 
+    # append to amis.txt file
+    if args.append:
+        with open(args.cloudformation_template) as f:
+            data = json.load(f)
+            amis = data.get('Mappings').get('AWSRegionOS2AMI')
+            amis.update(amis_json)
+            amis_json = amis
+            regions = sorted(amis_json)
+
     # convert json to txt
     amis_txt = convert_json_to_txt(regions=regions, amis_json=amis_json)