Release 1.5.2

Author: sean-smith

.travis.yml

@@ -10,6 +10,10 @@ python:
 
 install: 
   - cd cli
+  - if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then
+    pip install 'setuptools==39.2.0' --force-reinstall;
+    pip install -U enum34;
+    fi
   - if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; 
     then pip install -r requirements26.txt; 
     else pip install -r requirements.txt;
@@ -19,4 +23,8 @@ install:
 
 sudo: false
 
-script: sh tests/test.sh
+script:
+  - if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]];
+    then python cli/tests/cfncluster-unittest.py;
+    fi
+  - sh tests/test.sh

CHANGELOG.rst

@@ -2,6 +2,15 @@
 CHANGELOG
 =========
 
+1.5.2
+=====
+* feature:``cfncluster``: Added ClusterUser as a stack output. This makes it easier to get the username of the head node.
+* feature:``cfncluster``: Added `cfncluster ssh cluster_name`, this allows you to easily ssh into your clusters. It allows arbitrary command execution and extra ssh flags to be provided after the command. See https://cfncluster.readthedocs.io/en/latest/commands.html#ssh
+* change:``cfncluster``: Moved global cli flags to the command specific flags. For example `cfncluster --region us-east-1 create` now becomes `cfncluster create --region us-east-1`
+* bugfix:``cfncluster-cookbook``: Fix bug that prevented c5d/m5d instances from working
+* bugfix:``cfncluster-cookbook``: Set CPU as a consumable resource in slurm
+* bugfix:``cfncluster-node``: Fixed Slurm behavior to add CPU slots so multiple jobs can be scheduled on a single node
+
 1.5.1
 =====
 * change:``cfncluster``: Added "ec2:DescribeVolumes" permissions to

CONTRIBUTORS.txt

@@ -1,85 +1,85 @@
 # alinux
-ap-northeast-1: ami-0dde39e2d5b865e22
-ap-northeast-2: ami-00ff07cc9e3be7301
-ap-northeast-3: ami-00683abdf6e60ebcd
-ap-south-1: ami-0aa10b91c994a481d
-ap-southeast-1: ami-05698f82a71d681a5
-ap-southeast-2: ami-01da8b5a3a0212da1
-ca-central-1: ami-0845890c227805aeb
-eu-central-1: ami-54cffcbf
-eu-west-1: ami-08cb4b06d393675b0
-eu-west-2: ami-0cb0bfe1b8e796cd7
-eu-west-3: ami-0d277eafd4f569bc7
-sa-east-1: ami-056a9e6c13067d81f
-us-east-1: ami-0f4de43b53d2f8210
-us-east-2: ami-00c76d9774b0e366b
-us-west-1: ami-0cf41903fbdc094cc
-us-west-2: ami-03eee80fee70a432d
+ap-northeast-1: ami-05468a90639fdc483
+ap-northeast-2: ami-0b4d5a4a999581db1
+ap-northeast-3: ami-0594d5d373771f10a
+ap-south-1: ami-0ec7be770210af7c0
+ap-southeast-1: ami-02181eb4bbe285f28
+ap-southeast-2: ami-03199596cd474bb05
+ca-central-1: ami-099a5f4b65d56215e
+eu-central-1: ami-00e39ce170dbd6e25
+eu-west-1: ami-059b3db381c18305a
+eu-west-2: ami-0df738bfb4cf98f4e
+eu-west-3: ami-0b1472bf5fed029bd
+sa-east-1: ami-0a390e1998fb6b22f
+us-east-1: ami-0fd6854902340b407
+us-east-2: ami-04fc90a3f60cb3dc7
+us-west-1: ami-0054acc8361046ccc
+us-west-2: ami-01382f63e2667a7d1
 # centos6
-ap-northeast-1: ami-04d400f3c012a86ae
-ap-northeast-2: ami-068f5c3e07b48f123
-ap-northeast-3: ami-094d5e2e854fb015e
-ap-south-1: ami-0c8a6bf281179c3ba
-ap-southeast-1: ami-09555649a6bee7b69
-ap-southeast-2: ami-0610ffef47d7427d9
-ca-central-1: ami-0a36ee35b01f0e7c5
-eu-central-1: ami-89b38062
-eu-west-1: ami-00900f4e20bdb9f6d
-eu-west-2: ami-092aee04074dfd1f4
-eu-west-3: ami-042907d2cd70ecb40
-sa-east-1: ami-09d0f784c22548994
-us-east-1: ami-090b5303b54f5b35a
-us-east-2: ami-036cfbf5fc84c861e
-us-west-1: ami-0cc517b5b49ed6862
-us-west-2: ami-0e1ae8c0ca706fe64
+ap-northeast-1: ami-0603daa665c23cf74
+ap-northeast-2: ami-06ddec14c10c7de14
+ap-northeast-3: ami-0e718e0794e781ead
+ap-south-1: ami-043871a624a63c4b3
+ap-southeast-1: ami-07df16800a8f39379
+ap-southeast-2: ami-09b23d993fbc703d9
+ca-central-1: ami-0af06655a7341c2b5
+eu-central-1: ami-0362d5eb8ec477a30
+eu-west-1: ami-04c68c9303ee0165c
+eu-west-2: ami-0d3a080fd047d6f03
+eu-west-3: ami-0ce8fcafcd4cb66ba
+sa-east-1: ami-010873231de6443db
+us-east-1: ami-0051f4828527649d0
+us-east-2: ami-0a585da16a01304bb
+us-west-1: ami-0fcbc3afb66cfcffd
+us-west-2: ami-0430c74956ce8e41a
 # centos7
-ap-northeast-1: ami-0906ec2a1fce4fe7a
-ap-northeast-2: ami-098685fc947384e15
-ap-northeast-3: ami-07d36e7c754056e76
-ap-south-1: ami-0fce8c70d048719f4
-ap-southeast-1: ami-030771765a1101265
-ap-southeast-2: ami-0930016c10f90c7f0
-ca-central-1: ami-05652bb89f388a076
-eu-central-1: ami-dbc0f330
-eu-west-1: ami-0433c843433f06547
-eu-west-2: ami-07080ddf1e2406ca2
-eu-west-3: ami-062bd0a49558d0ba0
-sa-east-1: ami-005a0e54c37bc2a7b
-us-east-1: ami-01a681672ca8e9de1
-us-east-2: ami-07af67d83c0ae927c
-us-west-1: ami-00f4867c7e867784b
-us-west-2: ami-03ac1fbbf78947521
+ap-northeast-1: ami-03e99430742e7d834
+ap-northeast-2: ami-0a3ad3da5f45318a4
+ap-northeast-3: ami-0804a0d2dcf1e675f
+ap-south-1: ami-06b1a64ab30690802
+ap-southeast-1: ami-06eec4db7a3965f04
+ap-southeast-2: ami-0ab1def1c27a54ada
+ca-central-1: ami-089b3d6d4ef8c5772
+eu-central-1: ami-0842b79de396801b5
+eu-west-1: ami-019ecc5629a990951
+eu-west-2: ami-03e914c8e1b836bcf
+eu-west-3: ami-099b3883ce5e219b3
+sa-east-1: ami-0edbfab1cbd73ac82
+us-east-1: ami-00976a3c38aaac1a2
+us-east-2: ami-0f7dbb8d94e96b0ab
+us-west-1: ami-0fae379328b9061bc
+us-west-2: ami-0df85f5bc2d980e57
 # ubuntu1404
-ap-northeast-1: ami-04a90ea564e67f222
-ap-northeast-2: ami-0255a27fa98c0528f
-ap-northeast-3: ami-0cca427720a003215
-ap-south-1: ami-014f89aad9256d002
-ap-southeast-1: ami-0ce7dce420b05943b
-ap-southeast-2: ami-0ea64377e3b3b90df
-ca-central-1: ami-0cd24474b7cea7208
-eu-central-1: ami-7ab18291
-eu-west-1: ami-0e1dae876983c1547
-eu-west-2: ami-0ea3003a9feecc513
-eu-west-3: ami-0605f0c49b21107f6
-sa-east-1: ami-09657aae9df227834
-us-east-1: ami-063f3448775e0f9eb
-us-east-2: ami-0fc2efe14121a0f34
-us-west-1: ami-0c9cdda5cdea561c3
-us-west-2: ami-0bf2e2047ba8e44f2
+ap-northeast-1: ami-068214c5cfb518070
+ap-northeast-2: ami-0d55e5524b8536518
+ap-northeast-3: ami-0ad3400658fbc51fa
+ap-south-1: ami-0b54a1bd3d82929e6
+ap-southeast-1: ami-0d3382bea35764ad2
+ap-southeast-2: ami-0a2a8aa487ce39e8c
+ca-central-1: ami-0bc510154aa82211d
+eu-central-1: ami-0174c3f5501fc441b
+eu-west-1: ami-01848c6ae1a9c6460
+eu-west-2: ami-0b867b5579d0f7a08
+eu-west-3: ami-0e8f93d19635c5d49
+sa-east-1: ami-048422e42509a1596
+us-east-1: ami-0ba2fcbf9fa378762
+us-east-2: ami-05aea9ea89d577194
+us-west-1: ami-0089ace30f2612d13
+us-west-2: ami-06142a55749b7912b
 # ubuntu1604
-ap-northeast-1: ami-0b98c32fcc2c04bc2
-ap-northeast-2: ami-0dc1b8292d6a0ab0b
-ap-northeast-3: ami-0681a1857c07f6497
-ap-south-1: ami-05247ccd8ba04d440
-ap-southeast-1: ami-0e2b3651fe0a5734f
-ap-southeast-2: ami-0ee926ceddd54fc92
-ca-central-1: ami-01d71e35696000851
-eu-central-1: ami-dec0f335
-eu-west-1: ami-075d0841a0892ca5c
-eu-west-2: ami-06fa81f4e8255415a
-eu-west-3: ami-0fbfe76a8613c0d41
-sa-east-1: ami-033feb8d267b3a76e
-us-east-1: ami-003db261b3fbed1e3
-us-east-2: ami-05d8e0d6e52ecd2ed
-us-west-1: ami-0cd5cc8977ef144a5
-us-west-2: ami-0d3a3ff36f97c7ba4
+ap-northeast-1: ami-03f6879ede5f51a6f
+ap-northeast-2: ami-0036ef722f2d0a331
+ap-northeast-3: ami-0a8db47e6bc6f585b
+ap-south-1: ami-08870f6ce8227e90f
+ap-southeast-1: ami-04950204e84f8bf64
+ap-southeast-2: ami-02b4175018a85c03d
+ca-central-1: ami-04a4a63323b55af6e
+eu-central-1: ami-089e9903d9c3bee1b
+eu-west-1: ami-033b9190bab9d42aa
+eu-west-2: ami-049d2247a1c3d582c
+eu-west-3: ami-09c4322a65746f77b
+sa-east-1: ami-0740b75b4caaa3b77
+us-east-1: ami-05790a6a634720a0d
+us-east-2: ami-0a0fcd1219e40a8f5
+us-west-1: ami-056472d51cf2fed45
+us-west-2: ami-0702d59e57ada76b8

amis.txt

@@ -10,12 +10,12 @@
 # or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
 # OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and
 # limitations under the License.
-
 from builtins import str
 import sys
 import time
 import logging
 import boto3
+import os
 from botocore.exceptions import ClientError
 
 from . import cfnconfig
@@ -277,6 +277,31 @@ def poll_master_server_state(stack_name, config):
 
     return state
 
+def get_master_server_ip(stack_name, config):
+    ec2 = boto3.client('ec2', region_name=config.region,
+                       aws_access_key_id=config.aws_access_key_id,
+                       aws_secret_access_key=config.aws_secret_access_key)
+
+    master_id = get_master_server_id(stack_name, config)
+
+    try:
+        instance = ec2.describe_instances(InstanceIds=[master_id]) \
+            .get('Reservations')[0] \
+            .get('Instances')[0]
+        ip_address = instance.get('PublicIpAddress')
+        state = instance.get('State').get('Name')
+        if state != 'running' or ip_address is None:
+            logger.info("MasterServer: %s\nCannot get ip address." % state.upper)
+            sys.exit(1)
+        return ip_address
+    except ClientError as e:
+        logger.critical(e.response.get('Error').get('Message'))
+        sys.stdout.flush()
+        sys.exit(1)
+    except KeyboardInterrupt:
+        logger.info('\nExiting...')
+        sys.exit(0)
+
 def get_ec2_instances(stack, config):
     cfn = boto3.client('cloudformation', region_name=config.region,
                        aws_access_key_id=config.aws_access_key_id,
@@ -346,6 +371,56 @@ def instances(args):
     for instance in instances:
         print('%s         %s' % (instance[0],instance[1]))
 
+def get_head_user(parameters, template):
+    mappings = template.get("TemplateBody") \
+            .get("Mappings") \
+            .get("OSFeatures")
+    base_os =[i.get('ParameterValue') for i in parameters if i.get('ParameterKey') == "BaseOS"][0]
+    return mappings.get(base_os).get("User")
+
+def command(args, extra_args):
+    stack = ('cfncluster-' + args.cluster_name)
+    config = cfnconfig.CfnClusterConfig(args)
+    if args.command in config.aliases:
+        config_command = config.aliases[args.command]
+    else:
+        config_command = "ssh {CFN_USER}@{MASTER_IP} {ARGS}"
+
+    cfn = boto3.client('cloudformation', region_name=config.region,
+                       aws_access_key_id=config.aws_access_key_id,
+                       aws_secret_access_key=config.aws_secret_access_key)
+    try:
+        status = cfn.describe_stacks(StackName=stack).get("Stacks")[0].get('StackStatus')
+        invalid_status = ['DELETE_COMPLETE', 'DELETE_IN_PROGRESS']
+        if status in invalid_status:
+            logger.info("Stack status: %s. Cannot SSH while in %s" % (status, ' or '.join(invalid_status)))
+            sys.exit(1)
+        ip = get_master_server_ip(stack, config)
+        stack_result = cfn.describe_stacks(StackName=stack).get('Stacks')[0]
+        template = cfn.get_template(StackName=stack)
+        username = get_head_user(stack_result.get('Parameters'), template)
+
+        try:
+            from shlex import quote as cmd_quote
+        except ImportError:
+            from pipes import quote as cmd_quote
+
+        # build command
+        cmd = config_command.format(CFN_USER=username, MASTER_IP=ip, ARGS=' '.join(cmd_quote(str(e)) for e in extra_args))
+
+        # run command
+        if not args.dryrun:
+            os.system(cmd)
+        else:
+            logger.info(cmd)
+    except ClientError as e:
+            logger.critical(e.response.get('Error').get('Message'))
+            sys.stdout.flush()
+            sys.exit(1)
+    except KeyboardInterrupt:
+        logger.info('\nExiting...')
+        sys.exit(0)
+
 def status(args):
     stack = ('cfncluster-' + args.cluster_name)
     config = cfnconfig.CfnClusterConfig(args)
@@ -406,9 +481,12 @@ def delete(args):
                        aws_secret_access_key=config.aws_secret_access_key)
 
     try:
+        # delete_stack does not raise an exception if stack does not exist
+        # Use describe_stacks to explicitly check if the stack exists
+        cfn.describe_stacks(StackName=stack)
         cfn.delete_stack(StackName=stack)
-        status = cfn.describe_stacks(StackName=stack).get("Stacks")[0].get('StackStatus')
         saw_update = True
+        status = cfn.describe_stacks(StackName=stack).get("Stacks")[0].get('StackStatus')
         sys.stdout.write('\rStatus: %s' % status)
         sys.stdout.flush()
         logger.debug('Status: %s' % status)
@@ -439,4 +517,3 @@ def delete(args):
     except KeyboardInterrupt:
         logger.info('\nExiting...')
         sys.exit(0)
-

cli/cfncluster/cfncluster.py

@@ -52,7 +52,7 @@ def __init__(self, args):
         __args_func = self.args.func.__name__
 
         # Determine config file name based on args or default
-        if args.config_file is not None:
+        if hasattr(args, 'config_file') and args.config_file is not None:
             self.__config_file = args.config_file
         else:
             self.__config_file = os.path.expanduser(os.path.join('~', '.cfncluster', 'config'))
@@ -76,7 +76,7 @@ def __init__(self, args):
 
         # Determine the EC2 region to used used or default to us-east-1
         # Order is 1) CLI arg 2) AWS_DEFAULT_REGION env 3) Config file 4) us-east-1
-        if args.region:
+        if hasattr(args, 'region') and args.region:
             self.region = args.region
         else:
             if os.environ.get('AWS_DEFAULT_REGION'):
@@ -213,6 +213,10 @@ def __init__(self, args):
                 self.parameters.append((self.__vpc_options.get(key)[0],__temp__))
             except configparser.NoOptionError:
                 pass
+            except configparser.NoSectionError:
+                print("ERROR: VPC section [%s] used in [%s] section is not defined"
+                      % (self.__vpc_section, self.__cluster_section))
+                sys.exit(1)
 
         # Dictionary list of all cluster section options
         self.__cluster_options = dict(cluster_user=('ClusterUser', None), compute_instance_type=('ComputeInstanceType',None),
@@ -334,6 +338,13 @@ def __init__(self, args):
         except AttributeError:
             pass
 
+        # handle aliases
+        self.aliases = {}
+        self.__alias_section = 'aliases'
+        if __config.has_section(self.__alias_section):
+            for alias in __config.options(self.__alias_section):
+                self.aliases[alias] = __config.get(self.__alias_section, alias)
+
         # Handle extra parameters supplied on command-line
         try:
             if self.args.extra_parameters is not None: