Skip to content

Commit dd80dc6

Browse files
dougalbdougalb
committed
Merge pull request #32 from awslabs/develop
Merge develop with v0.0.21 release
2 parents 4f63455 + 918baea commit dd80dc6

File tree

11 files changed

+236
-31
lines changed

11 files changed

+236
-31
lines changed

‎CHANGELOG.rst

+7
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@
22
CHANGELOG
33
=========
44

5+
0.0.21
6+
=======
7+
* feature:``cfncluster``: Support for dedicated tenancy
8+
* feature:``cfncluster``: Support for customer provided KMS keys (EBS and ephemeral)
9+
* updates:``ami``: Pulled latest CentOS6 errata
10+
* feature:``cfncluster``: Support for M4 instances
11+
512
0.0.20
613
======
714
* feature:``cfncluster``: Support for D2 instances

‎amis.txt

+10-9
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
us-west-2 ami-39d1fb09
2-
eu-central-1 ami-36eed22b
3-
sa-east-1 ami-19890c04
4-
ap-northeast-1 ami-e051abe0
5-
eu-west-1 ami-6f305018
6-
us-east-1 ami-f07f4598
7-
us-west-1 ami-a703e1e3
8-
ap-southeast-2 ami-4709747d
9-
ap-southeast-1 ami-d88bb98a
1+
us-west-2 ami-b7bb8387
2+
eu-central-1 ami-9886bf85
3+
sa-east-1 ami-853fbf98
4+
ap-northeast-1 ami-10aa7310
5+
eu-west-1 ami-5d6d132a
6+
us-east-1 ami-77fe0b1c
7+
us-west-1 ami-036f8547
8+
ap-southeast-2 ami-8d6b10b7
9+
ap-southeast-1 ami-b2d7ede0
10+
us-gov-west-1 ami-3d82e21e

‎bootstrap/src/scripts/boot_as_master

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@ dev=$(stat /dev/disk/by-ebs-volumeid/${cfn_volume}|grep -- 'File:'|awk '{print $
6868
fs_type=$(blkid -o list /dev/$dev | grep -- "$dev" | awk '{print $2}')
6969
if [ "${fs_type}x" == "x" ]; then
7070
mkfs.xfs /dev/disk/by-ebs-volumeid/${cfn_volume} || RC=1
71+
fs_type="xfs"
7172
sleep 5
7273
fi
73-
fs_type=$(blkid -o list | grep -- "$dev" | awk '{print $2}')
7474
echo "/dev/disk/by-ebs-volumeid/${cfn_volume} ${cfn_shared_dir} $fs_type noatime,nodiratime,_netdev 0 0" >> /etc/fstab
7575
mkdir -p ${cfn_shared_dir} || RC=1
7676
mount -v ${cfn_shared_dir} || RC=1

‎bootstrap/src/scripts/functions.shlib

+11-1
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,17 @@ function setup_ephemeral_drives () {
111111
mkfs -q /dev/ram1 1024 || RC=1
112112
mkdir -p /root/keystore || RC=1
113113
mount /dev/ram1 /root/keystore || RC=1
114-
dd if=/dev/urandom of=/root/keystore/keyfile bs=1024 count=4 || RC=1
114+
if [ "${cfn_ephemeral_kms_key_id}" != "NONE" ]; then
115+
# Use KMS for keyfile
116+
instanceId=$(curl -fs http://169.254.169.254/latest/meta-data/instance-id)
117+
kms_array=($(aws --region ${cfn_region} kms generate-data-key --key-id ${cfn_ephemeral_kms_key_id} --number-of-bytes 1024 --encryption-context InstanceId=${instanceId} --output text))
118+
echo -n ${kms_array[0]} > /root/keystore/keyfile || RC=1
119+
echo -n ${kms_array[2]} > /root/ephemeral_ciphertext.blob || RC=1
120+
chmod 0400 /root/ephemeral_ciphertext.blob || RC=1
121+
else
122+
# Use urandom for keyfile
123+
dd if=/dev/urandom of=/root/keystore/keyfile bs=1024 count=4 || RC=1
124+
fi
115125
chmod 0400 /root/keystore/keyfile || RC=1
116126
cryptsetup -q luksFormat /dev/vg.01/lv_ephemeral /root/keystore/keyfile || RC=1
117127
cryptsetup -d /root/keystore/keyfile luksOpen /dev/vg.01/lv_ephemeral ephemeral_luks || RC=1

‎cli/cfncluster/cfncluster.py

+2-1
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,8 @@ def delete(args):
309309
aws_secret_access_key=config.aws_secret_access_key)
310310
try:
311311
cfnconn.delete_stack(stack)
312-
time.sleep(5)
312+
if not args.nowait:
313+
time.sleep(5)
313314
status = cfnconn.describe_stacks(stack)[0].stack_status
314315
sys.stdout.write('\rStatus: %s' % status)
315316
sys.stdout.flush()

‎cli/cfncluster/cfnconfig.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,8 @@ def __init__(self, args):
203203
encrypted_ephemeral=('EncryptedEphemeral',None),pre_install_args=('PreInstallArgs',None),
204204
post_install_args=('PostInstallArgs',None), s3_read_resource=('S3ReadResource',None),
205205
s3_read_write_resource=('S3ReadWriteResource',None),cwl_region=('CWLRegion',None),
206-
cwl_log_group=('CWLLogGroup',None),shared_dir=('SharedDir',None)
206+
cwl_log_group=('CWLLogGroup',None),shared_dir=('SharedDir',None),tenancy=('Tenancy',None),
207+
ephemeral_kms_key_id=('EphemeralKMSKeyId',None), cluster_ready=('ClusterReadyScript','URL')
207208
)
208209

209210
# Loop over all the cluster options and add define to parameters, raise Exception if defined but null
@@ -234,7 +235,7 @@ def __init__(self, args):
234235

235236
# Dictionary list of all EBS options
236237
self.__ebs_options = dict(ebs_snapshot_id=('EBSSnapshotId','EC2Snapshot'), volume_type=('VolumeType',None),
237-
volume_size=('VolumeSize',None),
238+
volume_size=('VolumeSize',None), ebs_kms_key_id=('EBSKMSKeyId', None),
238239
volume_iops=('VolumeIOPS',None), encrypted=('EBSEncryption',None))
239240

240241
try:

‎cli/setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ def read(fname):
2020
return open(os.path.join(os.path.dirname(__file__), fname)).read()
2121

2222
console_scripts = ['cfncluster = cfncluster.cli:main']
23-
version = "0.0.20"
23+
version = "0.0.99"
2424
requires = ['boto>=2.38']
2525

2626
if sys.version_info[:2] == (2, 6):

‎cloudformation/cfncluster.cfn.json

+160-14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"AWSTemplateFormatVersion" : "2010-09-09",
3-
"Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-20150415-0 cfncluster-0.0.20",
3+
"Description" : "AWS CloudFormation Sample Template cfncluster.cfn.json: Sample template showing an framework for deploying master + compute type clusters on AWS. **WARNING** This template creates AWS resources. You will be billed for the AWS resources used if you create a stack from this template. Version: ami-20150609-0 cfncluster-0.0.21",
44
"Parameters" : {
55
"KeyName" : {
66
"Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
@@ -46,7 +46,12 @@
4646
"d2.8xlarge",
4747
"d2.4xlarge",
4848
"d2.2xlarge",
49-
"d2.xlarge"
49+
"d2.xlarge",
50+
"m4.large",
51+
"m4.xlarge",
52+
"m4.2xlarge",
53+
"m4.4xlarge",
54+
"m4.10xlarge"
5055
]
5156
},
5257
"ComputeInstanceType" : {
@@ -89,7 +94,12 @@
8994
"d2.8xlarge",
9095
"d2.4xlarge",
9196
"d2.2xlarge",
92-
"d2.xlarge"
97+
"d2.xlarge",
98+
"m4.large",
99+
"m4.xlarge",
100+
"m4.2xlarge",
101+
"m4.4xlarge",
102+
"m4.10xlarge"
93103
]
94104
},
95105
"InitialQueueSize" : {
@@ -358,6 +368,30 @@
358368
"Description" : "CloudWatch Logs LogGroup",
359369
"Type" : "String",
360370
"Default" : "NONE"
371+
},
372+
"Tenancy" : {
373+
"Description" : "Type of placement requird in cfncluster, it can either be cluster or compute.",
374+
"Type" : "String",
375+
"Default" : "default",
376+
"AllowedValues" : [
377+
"default",
378+
"dedicated"
379+
]
380+
},
381+
"EBSKMSKeyId" : {
382+
"Description" : "KMS ARN for customer created master key, will be used for EBS encryption",
383+
"Type" : "String",
384+
"Default" : "NONE"
385+
},
386+
"EphemeralKMSKeyId" : {
387+
"Description" : "KMS ARN for customer created master key, will be used for ephemeral encryption",
388+
"Type" : "String",
389+
"Default" : "NONE"
390+
},
391+
"ClusterReadyScript" : {
392+
"Description" : "Cluster ready script URL. This is only on the MasterServer, when the cluster reaches CREATE_COMPLETE.",
393+
"Type" : "String",
394+
"Default" : "NONE"
361395
}
362396
},
363397
"Conditions" : {
@@ -598,6 +632,57 @@
598632
]
599633
}
600634
]
635+
},
636+
"UseEBSKMSKey" : {
637+
"Fn::And" : [
638+
{
639+
"Fn::Not" : [
640+
{
641+
"Fn::Equals" : [
642+
{
643+
"Ref" : "EBSKMSKeyId"
644+
},
645+
"NONE"
646+
]
647+
}
648+
]
649+
},
650+
{
651+
"Condition" : "UseEBSEncryption"
652+
}
653+
]
654+
},
655+
"UseEphemeralKMSKey" : {
656+
"Fn::And" : [
657+
{
658+
"Fn::Not" : [
659+
{
660+
"Fn::Equals" : [
661+
{
662+
"Ref" : "EphemeralKMSKeyId"
663+
},
664+
"NONE"
665+
]
666+
}
667+
]
668+
},
669+
{
670+
"Fn::Equals" : [
671+
{
672+
"Ref" : "EncryptedEphemeral"
673+
},
674+
"true"
675+
]
676+
}
677+
]
678+
},
679+
"UseDedicatedTenancy" : {
680+
"Fn::Equals" : [
681+
{
682+
"Ref" : "Tenancy"
683+
},
684+
"dedicated"
685+
]
601686
}
602687
},
603688
"Mappings" : {
@@ -725,38 +810,74 @@
725810
"t2.medium" : {
726811
"Arch" : "64HVM",
727812
"EBSOpt" : "False"
813+
},
814+
"d2.8xlarge" : {
815+
"Arch" : "64HVM",
816+
"EBSOpt" : "True"
817+
},
818+
"d2.4xlarge" : {
819+
"Arch" : "64HVM",
820+
"EBSOpt" : "True"
821+
},
822+
"d2.2xlarge" : {
823+
"Arch" : "64HVM",
824+
"EBSOpt" : "True"
825+
},
826+
"d2.xlarge" : {
827+
"Arch" : "64HVM",
828+
"EBSOpt" : "True"
829+
},
830+
"m4.10xlarge" : {
831+
"Arch" : "64HVM",
832+
"EBSOpt" : "True"
833+
},
834+
"m4.4xlarge" : {
835+
"Arch" : "64HVM",
836+
"EBSOpt" : "True"
837+
},
838+
"m4.2xlarge" : {
839+
"Arch" : "64HVM",
840+
"EBSOpt" : "True"
841+
},
842+
"m4.xlarge" : {
843+
"Arch" : "64HVM",
844+
"EBSOpt" : "True"
845+
},
846+
"m4.large" : {
847+
"Arch" : "64HVM",
848+
"EBSOpt" : "True"
728849
}
729850
},
730851
"AWSRegionOS2AMI" : {
731852
"us-west-2" : {
732-
"centos6" : "ami-39d1fb09"
853+
"centos6" : "ami-b7bb8387"
733854
},
734855
"eu-central-1" : {
735-
"centos6" : "ami-36eed22b"
856+
"centos6" : "ami-9886bf85"
736857
},
737858
"sa-east-1" : {
738-
"centos6" : "ami-19890c04"
859+
"centos6" : "ami-853fbf98"
739860
},
740861
"ap-northeast-1" : {
741-
"centos6" : "ami-e051abe0"
862+
"centos6" : "ami-10aa7310"
742863
},
743864
"eu-west-1" : {
744-
"centos6" : "ami-6f305018"
865+
"centos6" : "ami-5d6d132a"
745866
},
746867
"us-east-1" : {
747-
"centos6" : "ami-f07f4598"
868+
"centos6" : "ami-77fe0b1c"
748869
},
749870
"us-west-1" : {
750-
"centos6" : "ami-a703e1e3"
871+
"centos6" : "ami-036f8547"
751872
},
752873
"ap-southeast-2" : {
753-
"centos6" : "ami-4709747d"
874+
"centos6" : "ami-8d6b10b7"
754875
},
755876
"ap-southeast-1" : {
756-
"centos6" : "ami-d88bb98a"
877+
"centos6" : "ami-b2d7ede0"
757878
},
758879
"us-gov-west-1" : {
759-
"centos6" : "ami-a7f99884"
880+
"centos6" : "ami-3d82e21e"
760881
}
761882
},
762883
"ClusterUser" : {
@@ -1374,6 +1495,9 @@
13741495
"Ref" : "AWS::NoValue"
13751496
}
13761497
]
1498+
},
1499+
"Tenancy" : {
1500+
"Ref" : "Tenancy"
13771501
}
13781502
},
13791503
"Metadata" : {
@@ -1925,7 +2049,18 @@
19252049
}
19262050
]
19272051
},
1928-
"InstanceMonitoring" : "false"
2052+
"InstanceMonitoring" : "false",
2053+
"PlacementTenancy" : {
2054+
"Fn::If" : [
2055+
"UseDedicatedTenancy",
2056+
{
2057+
"Ref" : "Tenancy"
2058+
},
2059+
{
2060+
"Ref" : "AWS::NoValue"
2061+
}
2062+
]
2063+
}
19292064
},
19302065
"Metadata" : {
19312066
"Comment" : "cfncluster Compute server",
@@ -2439,6 +2574,17 @@
24392574
"Ref" : "AWS::NoValue"
24402575
}
24412576
]
2577+
},
2578+
"KmsKeyId" : {
2579+
"Fn::If" : [
2580+
"UseEBSKMSKey",
2581+
{
2582+
"Ref" : "EBSKMSKeyId"
2583+
},
2584+
{
2585+
"Ref" : "AWS::NoValue"
2586+
}
2587+
]
24422588
}
24432589
}
24442590
},

‎docs/source/ami_customization.rst

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
.. _ami_customization:
2+
3+
.. toctree::
4+
:maxdepth: 2
5+
6+
################################
7+
Building a custom CfnCluster AMI
8+
################################
9+
10+
.. warning::
11+
Building a custom AMI is not the recomended approach for customizing CfnCluster.
12+
13+
Once you build your own AMI, you will no longer receive updates or bug fixes with future releases of CfnCluster. You will need to repeat the steps used to create your custom AMI with each new CfnCluster release.
14+
15+
Before reading any further, take a look at the :doc:`pre_post_install` section of the documentation to determine if the modifications you wish to make can be scripted and supported with future CfnCluster releases
16+
17+
While not ideal, there are a number of scenarios where building a custom AMI for CfnCluster is necessary. This tutorial will guide you through the process.
18+
19+
How to customize the CfnCluster AMI
20+
===================================
21+
22+
The base CfnCluster AMI is often updated with new releases. This AMI has all of the components required for CfnCluster to function installed and configured. If you wish to customize an AMI for CfnCluster, you must start with this as the base.
23+
24+
#. Find the AMI which corresponds with the region you will be utilizing in the list here: https://github.com/awslabs/cfncluster/blob/master/amis.txt.
25+
#. Within the EC2 Console, choose "Launch Instance".
26+
#. Navigate to "Community AMIs", and enter the AMI id for your region into the search box.
27+
#. Select the AMI, choose your instance type and properties, and launch your instance.
28+
#. Log into your instance using the ec2-user and your SSH key.
29+
#. Customize your instance as required
30+
#. Run the following command to prepare your instance for AMI creation::
31+
32+
sudo /usr/local/sbin/ami_cleanup.sh
33+
34+
#. Stop the instance
35+
#. Create a new AMI from the instance
36+
#. Enter the AMI id in the :ref:`custom_ami_section` field within your cluster configuration.

0 commit comments

Comments
 (0)