Skip to content

Commit a846029

Browse files
sean-smithsean-smith
authored
Release 1.5.4
Release 1.5.4
2 parents 687118b + 8e9ac16 commit a846029

19 files changed

+448
-294
lines changed

‎.travis.yml

+5-2
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ install:
2424
sudo: false
2525

2626
script:
27-
- if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]];
28-
then python cli/tests/cfncluster-unittest.py;
27+
- if [[ $TRAVIS_PYTHON_VERSION != 2.6 ]]; then
28+
pip install 'boto3==1.7.84' --force-reinstall;
29+
pip install moto
30+
python cli/tests/cfncluster-unittest.py;
31+
pip install boto3 --upgrade;
2932
fi
3033
- sh tests/test.sh

‎CHANGELOG.rst

+7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22
CHANGELOG
33
=========
44

5+
1.5.4
6+
=====
7+
* Add option to disable ganglia `extra_json = { "cfncluster" : { "ganglia_enabled" : "no" } }`
8+
* Fix `cfncluster update` bug
9+
* Set SGE Accounting summary to be true, this reports a single accounting record for a mpi job
10+
* Upgrade cfncluster-node to Boto3
11+
512
1.5.3
613
=====
714
* Add support for GovCloud, us-gov-west-1 region

‎README.rst

+5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@ Please open a GitHub issue for any feedback or issues:
2828
https://github.com/awslabs/cfncluster. There is also an active AWS
2929
HPC forum which may be helpful:https://forums.aws.amazon.com/forum.jspa?forumID=192.
3030

31+
CfnCluster 1.6 IAM Change
32+
=========================
33+
Between CfnCluster 1.5.3 and 1.6.0 we made a change to the CfnClusterInstancePolicy that adds “s3:GetObject” permissions on objects in <REGION>-cfncluster bucket.
34+
If you’re using a custom policy (e.g. you specify "ec2_iam_role" in your config) be sure it includes this new permission. See https://cfncluster.readthedocs.io/en/latest/iam.html
35+
3136
CfnCluster 1.5 IAM Change
3237
=========================
3338
Between CfnCluster 1.4.2 and 1.5.0 we made a change to the CfnClusterInstancePolicy that adds “ec2:DescribeVolumes” permissions. If you’re using a custom policy (e.g. you specify "ec2_iam_role" in your config) be sure it includes this new permission. See https://cfncluster.readthedocs.io/en/latest/iam.html

‎amis.txt

+83-83
Original file line numberDiff line numberDiff line change
@@ -1,88 +1,88 @@
11
# alinux
2-
ap-northeast-1: ami-05468a90639fdc483
3-
ap-northeast-2: ami-0b4d5a4a999581db1
4-
ap-northeast-3: ami-0594d5d373771f10a
5-
ap-south-1: ami-0ec7be770210af7c0
6-
ap-southeast-1: ami-02181eb4bbe285f28
7-
ap-southeast-2: ami-03199596cd474bb05
8-
ca-central-1: ami-099a5f4b65d56215e
9-
eu-central-1: ami-00e39ce170dbd6e25
10-
eu-west-1: ami-059b3db381c18305a
11-
eu-west-2: ami-0df738bfb4cf98f4e
12-
eu-west-3: ami-0b1472bf5fed029bd
13-
sa-east-1: ami-0a390e1998fb6b22f
14-
us-east-1: ami-0fd6854902340b407
15-
us-east-2: ami-04fc90a3f60cb3dc7
16-
us-gov-west-1: ami-af2cbece
17-
us-west-1: ami-0054acc8361046ccc
18-
us-west-2: ami-01382f63e2667a7d1
2+
ap-northeast-1: ami-00047986f877e6760
3+
ap-northeast-2: ami-0ca5afbc9d0b20f3d
4+
ap-northeast-3: ami-0b3af1c2fe35b1ae4
5+
ap-south-1: ami-0250a8315b099785d
6+
ap-southeast-1: ami-038312aa862774599
7+
ap-southeast-2: ami-0dce078cd2c343a5c
8+
ca-central-1: ami-04e27bd54e171fbad
9+
eu-central-1: ami-075bf1c36cf6b855f
10+
eu-west-1: ami-08d507a759b9ee0e4
11+
eu-west-2: ami-03ddfb1462e2c92fd
12+
eu-west-3: ami-0e72015cf2b21159f
13+
sa-east-1: ami-02bd755bfe8704cef
14+
us-east-1: ami-07f61c3e18538c70c
15+
us-east-2: ami-096b7925b1f9c8cbb
16+
us-gov-west-1: ami-cfd34dae
17+
us-west-1: ami-04794da830cb9dc86
18+
us-west-2: ami-03f86a2920360dfbe
1919
# centos6
20-
ap-northeast-1: ami-0603daa665c23cf74
21-
ap-northeast-2: ami-06ddec14c10c7de14
22-
ap-northeast-3: ami-0e718e0794e781ead
23-
ap-south-1: ami-043871a624a63c4b3
24-
ap-southeast-1: ami-07df16800a8f39379
25-
ap-southeast-2: ami-09b23d993fbc703d9
26-
ca-central-1: ami-0af06655a7341c2b5
27-
eu-central-1: ami-0362d5eb8ec477a30
28-
eu-west-1: ami-04c68c9303ee0165c
29-
eu-west-2: ami-0d3a080fd047d6f03
30-
eu-west-3: ami-0ce8fcafcd4cb66ba
31-
sa-east-1: ami-010873231de6443db
32-
us-east-1: ami-0051f4828527649d0
33-
us-east-2: ami-0a585da16a01304bb
34-
us-west-1: ami-0fcbc3afb66cfcffd
35-
us-west-2: ami-0430c74956ce8e41a
20+
ap-northeast-1: ami-00c9ad360d255b462
21+
ap-northeast-2: ami-08e84ffe9aead7873
22+
ap-northeast-3: ami-056531169587ffc23
23+
ap-south-1: ami-03a57d339b58b2f23
24+
ap-southeast-1: ami-00063e5635939c9d1
25+
ap-southeast-2: ami-015e730525001bf53
26+
ca-central-1: ami-0fea713db092ac77f
27+
eu-central-1: ami-0f6b91d011f44aaf6
28+
eu-west-1: ami-029accac823a2a844
29+
eu-west-2: ami-032b6b895dab544c6
30+
eu-west-3: ami-039be4bd46c878e2c
31+
sa-east-1: ami-0e507bdec09b8a091
32+
us-east-1: ami-091bbc83b27f92634
33+
us-east-2: ami-0fa89e68b0222f50b
34+
us-west-1: ami-06f3909be633646b9
35+
us-west-2: ami-087d4f29315fd19d2
3636
# centos7
37-
ap-northeast-1: ami-03e99430742e7d834
38-
ap-northeast-2: ami-0a3ad3da5f45318a4
39-
ap-northeast-3: ami-0804a0d2dcf1e675f
40-
ap-south-1: ami-06b1a64ab30690802
41-
ap-southeast-1: ami-06eec4db7a3965f04
42-
ap-southeast-2: ami-0ab1def1c27a54ada
43-
ca-central-1: ami-089b3d6d4ef8c5772
44-
eu-central-1: ami-0842b79de396801b5
45-
eu-west-1: ami-019ecc5629a990951
46-
eu-west-2: ami-03e914c8e1b836bcf
47-
eu-west-3: ami-099b3883ce5e219b3
48-
sa-east-1: ami-0edbfab1cbd73ac82
49-
us-east-1: ami-00976a3c38aaac1a2
50-
us-east-2: ami-0f7dbb8d94e96b0ab
51-
us-west-1: ami-0fae379328b9061bc
52-
us-west-2: ami-0df85f5bc2d980e57
37+
ap-northeast-1: ami-09dc3b175ed2905f8
38+
ap-northeast-2: ami-03f494fed9f0f9c98
39+
ap-northeast-3: ami-04094dd3b4013b75f
40+
ap-south-1: ami-0c20297a2c00b0528
41+
ap-southeast-1: ami-0f610508513f2012f
42+
ap-southeast-2: ami-09d0b89439c0c1b65
43+
ca-central-1: ami-0362a3ef26ab5dfce
44+
eu-central-1: ami-03d5519f5f4e7619f
45+
eu-west-1: ami-0575304add4986551
46+
eu-west-2: ami-08263362b21b98916
47+
eu-west-3: ami-0befb803ea7ce1b83
48+
sa-east-1: ami-0860c11596f954bbc
49+
us-east-1: ami-06c93be112f880acd
50+
us-east-2: ami-0493261a1b2adfd27
51+
us-west-1: ami-0af95249ebb8de7a5
52+
us-west-2: ami-0a259cf8220fabdcb
5353
# ubuntu1404
54-
ap-northeast-1: ami-068214c5cfb518070
55-
ap-northeast-2: ami-0d55e5524b8536518
56-
ap-northeast-3: ami-0ad3400658fbc51fa
57-
ap-south-1: ami-0b54a1bd3d82929e6
58-
ap-southeast-1: ami-0d3382bea35764ad2
59-
ap-southeast-2: ami-0a2a8aa487ce39e8c
60-
ca-central-1: ami-0bc510154aa82211d
61-
eu-central-1: ami-0174c3f5501fc441b
62-
eu-west-1: ami-01848c6ae1a9c6460
63-
eu-west-2: ami-0b867b5579d0f7a08
64-
eu-west-3: ami-0e8f93d19635c5d49
65-
sa-east-1: ami-048422e42509a1596
66-
us-east-1: ami-0ba2fcbf9fa378762
67-
us-east-2: ami-05aea9ea89d577194
68-
us-gov-west-1: ami-4aca572b
69-
us-west-1: ami-0089ace30f2612d13
70-
us-west-2: ami-06142a55749b7912b
54+
ap-northeast-1: ami-09dc9e9da730c0248
55+
ap-northeast-2: ami-06346bcc8b825a458
56+
ap-northeast-3: ami-0ae6e02016436404a
57+
ap-south-1: ami-0e4294a5b9d5de425
58+
ap-southeast-1: ami-01e6359021e3bac23
59+
ap-southeast-2: ami-0cfebd51a3470f1d4
60+
ca-central-1: ami-0097472502a04776d
61+
eu-central-1: ami-0a82ac2872821deef
62+
eu-west-1: ami-0d6c90f092266b787
63+
eu-west-2: ami-0f23dffc4db122b53
64+
eu-west-3: ami-02b6ca8fd14f58265
65+
sa-east-1: ami-04e943b561cd32659
66+
us-east-1: ami-0920fa86d5a85a07a
67+
us-east-2: ami-0d63a602f26709e70
68+
us-gov-west-1: ami-c2d34da3
69+
us-west-1: ami-06fe4f837d8715dbb
70+
us-west-2: ami-01170828a21d86a72
7171
# ubuntu1604
72-
ap-northeast-1: ami-03f6879ede5f51a6f
73-
ap-northeast-2: ami-0036ef722f2d0a331
74-
ap-northeast-3: ami-0a8db47e6bc6f585b
75-
ap-south-1: ami-08870f6ce8227e90f
76-
ap-southeast-1: ami-04950204e84f8bf64
77-
ap-southeast-2: ami-02b4175018a85c03d
78-
ca-central-1: ami-04a4a63323b55af6e
79-
eu-central-1: ami-089e9903d9c3bee1b
80-
eu-west-1: ami-033b9190bab9d42aa
81-
eu-west-2: ami-049d2247a1c3d582c
82-
eu-west-3: ami-09c4322a65746f77b
83-
sa-east-1: ami-0740b75b4caaa3b77
84-
us-east-1: ami-05790a6a634720a0d
85-
us-east-2: ami-0a0fcd1219e40a8f5
86-
us-gov-west-1: ami-3dc8555c
87-
us-west-1: ami-056472d51cf2fed45
88-
us-west-2: ami-0702d59e57ada76b8
72+
ap-northeast-1: ami-0b0bf645cde38fe52
73+
ap-northeast-2: ami-0b2c6c9e988689d7c
74+
ap-northeast-3: ami-0dae0e941e3bce10b
75+
ap-south-1: ami-011d655005fe854e9
76+
ap-southeast-1: ami-03428422e2cb20c9f
77+
ap-southeast-2: ami-0bbef9e2b1170757f
78+
ca-central-1: ami-0916119fde83c29b7
79+
eu-central-1: ami-003273c4072efbe43
80+
eu-west-1: ami-009af0d819973e1d6
81+
eu-west-2: ami-029f01bbfec52a10e
82+
eu-west-3: ami-0ae405adcb3d1477c
83+
sa-east-1: ami-0b0057ecd5c465d09
84+
us-east-1: ami-0dfb1662fcc845c55
85+
us-east-2: ami-01bd1361428c384c3
86+
us-gov-west-1: ami-0fd6486e
87+
us-west-1: ami-0e144dd24634766a2
88+
us-west-2: ami-02087bf0ff99cc127

‎cli/README

+12-11
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,27 @@
11
cfncluster is an example framework for launching clusters.
22

3-
usage: cfncluster [-h] [--config CONFIG_FILE] [--region REGION] [--nowait]
4-
{create,update,delete,status,list,instances,version}
3+
usage: cfncluster [-h]
4+
{create,update,delete,start,stop,status,list,instances,ssh,configure,version}
55
...
66

7-
cfncluster is the a tool to launch and manage cluster.
7+
cfncluster is a tool to launch and manage a cluster.
88

99
positional arguments:
10-
{create,update,stop,delete,status,list,instances,sshmaster}
10+
{create,update,delete,start,stop,status,list,instances,ssh,configure,version}
1111
create creates a cluster
1212
update update a running cluster
1313
delete delete a cluster
14+
start start the compute fleet that has been stopped
15+
stop stop the compute fleet, but leave the master server
16+
running for debugging/development
1417
status pull the current status of the cluster
1518
list display a list of stacks associated with cfncluster
1619
instances display a list of all instances in a cluster
17-
version display the version of cfncluster cli
20+
ssh connect to the master server using SSH
21+
configure creating initial cfncluster configuration
22+
version display version of cfncluster
1823

1924
optional arguments:
2025
-h, --help show this help message and exit
21-
--config CONFIG_FILE, -c CONFIG_FILE
22-
specify a alternative config file
23-
--region REGION, -r REGION
24-
specify a specific region to connect to
25-
--nowait, -nw do not wait for stack events, after executing stack
26-
command
26+
27+
For command specific flags run cfncluster [command] --help

‎cli/cfncluster/cfncluster.py

+14-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import logging
1717
import boto3
1818
import os
19+
import json
1920
from botocore.exceptions import ClientError
2021

2122
from . import cfnconfig
@@ -103,7 +104,10 @@ def create(args):
103104
event.get('ResourceStatusReason')))
104105
logger.info('')
105106
outputs = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('Outputs', [])
107+
ganglia_enabled = is_ganglia_enabled(config.parameters)
106108
for output in outputs:
109+
if not ganglia_enabled and output.get('OutputKey').startswith('Ganglia'):
110+
continue
107111
logger.info("%s: %s" % (output.get('OutputKey'), output.get('OutputValue')))
108112
else:
109113
status = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('StackStatus')
@@ -119,6 +123,14 @@ def create(args):
119123
logger.critical(e)
120124
sys.exit(1)
121125

126+
def is_ganglia_enabled(parameters):
127+
extra_json = dict(filter(lambda x: x[0] == 'ExtraJson', parameters))
128+
try:
129+
extra_json = json.loads(extra_json.get('ExtraJson')).get('cfncluster')
130+
return extra_json.get('ganglia_enabled') == 'yes'
131+
except:
132+
pass
133+
return False
122134

123135
def update(args):
124136
logger.info('Updating: %s' % (args.cluster_name))
@@ -139,7 +151,7 @@ def update(args):
139151
desired_capacity = asg.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])\
140152
.get('AutoScalingGroups')[0]\
141153
.get('DesiredCapacity')
142-
config.parameters.append(('InitialQueueSize', desired_capacity))
154+
config.parameters.append(('InitialQueueSize', str(desired_capacity)))
143155

144156
# Get the MasterSubnetId and use it to determine AvailabilityZone
145157
try:
@@ -162,7 +174,7 @@ def update(args):
162174
try:
163175
logger.debug((config.template_url, config.parameters))
164176

165-
cfn_params = [{'ParameterKey': param[0], 'ParameterValue': str(param[1])} for param in config.parameters]
177+
cfn_params = [{'ParameterKey': param[0], 'ParameterValue': param[1]} for param in config.parameters]
166178
cfn.update_stack(StackName=stack_name,TemplateURL=config.template_url,
167179
Parameters=cfn_params, Capabilities=capabilities)
168180
status = cfn.describe_stacks(StackName=stack_name).get("Stacks")[0].get('StackStatus')

‎cli/cfncluster/cli.py

+14-13
Original file line numberDiff line numberDiff line change
@@ -141,17 +141,17 @@ def main():
141141
addarg_nowait(pdelete)
142142
pdelete.set_defaults(func=delete)
143143

144-
pstart = subparsers.add_parser('start', help='start the compute-fleet that has been stopped')
144+
pstart = subparsers.add_parser('start', help='start the compute fleet that has been stopped')
145145
pstart.add_argument("cluster_name", type=str, default=None,
146-
help='starts the compute-fleet of the provided cluster name.')
146+
help='starts the compute fleet of the provided cluster name.')
147147
addarg_config(pstart)
148148
addarg_region(pstart)
149149
pstart.set_defaults(func=start)
150150

151-
pstop = subparsers.add_parser('stop', help='stop the compute-fleet, but leave the MasterServer running for '
151+
pstop = subparsers.add_parser('stop', help='stop the compute fleet, but leave the master server running for '
152152
'debugging/development')
153153
pstop.add_argument("cluster_name", type=str, default=None,
154-
help='stops the compute-fleet of the provided cluster name.')
154+
help='stops the compute fleet of the provided cluster name.')
155155
addarg_config(pstop)
156156
addarg_region(pstop)
157157
pstop.set_defaults(func=stop)
@@ -176,22 +176,23 @@ def main():
176176
addarg_region(pinstances)
177177
pinstances.set_defaults(func=instances)
178178

179+
pssh = subparsers.add_parser('ssh', help='connect to the master server using SSH',
180+
description='run ssh command with username and ip address pre-filled. ' \
181+
'Arbitrary arguments are appended to the end of the ssh commmand. ' \
182+
'This command may be customized in the aliases section of the config file.')
183+
pssh.add_argument("cluster_name", type=str, default=None,
184+
help='name of the cluster to set variables for.')
185+
pssh.add_argument("--dryrun", "-d", action='store_true', dest="dryrun", default=False,
186+
help='print command and exit.')
187+
pssh.set_defaults(func=command)
188+
179189
pconfigure = subparsers.add_parser('configure', help='creating initial cfncluster configuration')
180190
addarg_config(pconfigure)
181191
pconfigure.set_defaults(func=configure)
182192

183193
pversion = subparsers.add_parser('version', help='display version of cfncluster')
184194
pversion.set_defaults(func=version)
185195

186-
pssh = subparsers.add_parser('ssh', description='run ssh command with username and ip address pre-filled. ' \
187-
'Arbitrary arguments are appended to the end of the ssh commmand. ' \
188-
'This command may be customized in the aliases section of the config file.')
189-
pssh.add_argument("cluster_name", type=str, default=None,
190-
help='name of the cluster to set variables for.')
191-
pssh.add_argument("--dryrun", "-d", action='store_true', dest="dryrun", default=False,
192-
help='print command and exit.')
193-
pssh.set_defaults(func=command)
194-
195196
args, extra_args = parser.parse_known_args()
196197
logger.debug(args)
197198
if args.func.__name__ == 'command':

‎cli/cfncluster/config_sanity.py

+4-1
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,17 @@ def check_resource(region, aws_access_key_id, aws_secret_access_key, resource_ty
4040
aws_secret_access_key=aws_secret_access_key)
4141

4242
arn = iam.get_role(RoleName=resource_value).get('Role').get('Arn')
43-
accountid = boto3.client('sts').get_caller_identity().get('Account')
43+
accountid = boto3.client('sts', region_name=region,
44+
aws_access_key_id=aws_access_key_id,
45+
aws_secret_access_key=aws_secret_access_key).get_caller_identity().get('Account')
4446

4547
iam_policy = [(['ec2:DescribeVolumes', 'ec2:AttachVolume', 'ec2:DescribeInstanceAttribute', 'ec2:DescribeInstanceStatus', 'ec2:DescribeInstances'], "*"),
4648
(['dynamodb:ListTables'], "*"),
4749
(['sqs:SendMessage', 'sqs:ReceiveMessage', 'sqs:ChangeMessageVisibility', 'sqs:DeleteMessage', 'sqs:GetQueueUrl'], "arn:aws:sqs:%s:%s:cfncluster-*" % (region, accountid)),
4850
(['autoscaling:DescribeAutoScalingGroups', 'autoscaling:TerminateInstanceInAutoScalingGroup', 'autoscaling:SetDesiredCapacity'], "*"),
4951
(['cloudwatch:PutMetricData'], "*"),
5052
(['dynamodb:PutItem', 'dynamodb:Query', 'dynamodb:GetItem', 'dynamodb:DeleteItem', 'dynamodb:DescribeTable'], "arn:aws:dynamodb:%s:%s:table/cfncluster-*" % (region, accountid)),
53+
(['s3:GetObject'], "arn:aws:s3:::%s-cfncluster/*" % region),
5154
(['sqs:ListQueues'], "*"),
5255
(['logs:*'], "arn:aws:logs:*:*:*")]
5356

0 commit comments

Comments
 (0)