provider/ec2: update default controller instance

In a VPC account, we'll create a t2.medium for
controller instances by default now. Non-controller
instances are unchanged for now, and remain m3.medium
by default. Controller instance root disk now defaults
to 32GiB, but can be overridden with constraints.

There is a fix to the ec2instancetypes generator:
we were expecting "currentGeneration" to be present
in the attributes always, but it is not there for
some. We assume they are current if it is missing,
which is the case with the ones that have it missing
currently.

Fixes https://bugs.launchpad.net/juju/+bug/1535838

constraints/constraints.go

@@ -110,6 +110,24 @@ func (v *Value) HasArch() bool {
 	return v.Arch != nil && *v.Arch != ""
 }
 
+// HasMem returns true if the constraints.Value specifies a minimum amount
+// of memory.
+func (v *Value) HasMem() bool {
+	return v.Mem != nil && *v.Mem > 0
+}
+
+// HasCpuPower returns true if the constraints.Value specifies a minimum amount
+// of CPU power.
+func (v *Value) HasCpuPower() bool {
+	return v.CpuPower != nil && *v.CpuPower > 0
+}
+
+// HasCpuCores returns true if the constraints.Value specifies a minimum number
+// of CPU cores.
+func (v *Value) HasCpuCores() bool {
+	return v.CpuCores != nil && *v.CpuCores > 0
+}
+
 // HasInstanceType returns true if the constraints.Value specifies an instance type.
 func (v *Value) HasInstanceType() bool {
 	return v.InstanceType != nil && *v.InstanceType != ""

dependencies.tsv

@@ -67,7 +67,7 @@ golang.org/x/net	git	ea47fc708ee3e20177f3ca3716217c4ab75942cb	2015-08-29T23:03:1
 golang.org/x/oauth2	git	11c60b6f71a6ad48ed6f93c65fa4c6f9b1b5b46a	2015-03-25T02:00:22Z
 google.golang.org/api	git	0d3983fb069cb6651353fc44c5cb604e263f2a93	2014-12-10T23:51:26Z
 google.golang.org/cloud	git	f20d6dcccb44ed49de45ae3703312cb46e627db1	2015-03-19T22:36:35Z
-gopkg.in/amz.v3	git	a651c43e72df7778b14ac6b54e5ac119d32b1263	2016-04-20T02:16:08Z
+gopkg.in/amz.v3	git	18899065239e006cc73b0e66800c98c2ce4eee50	2016-10-06T07:29:34Z
 gopkg.in/check.v1	git	4f90aeace3a26ad7021961c297b22c42160c7b25	2016-01-05T16:49:36Z
 gopkg.in/errgo.v1	git	66cb46252b94c1f3d65646f54ee8043ab38d766c	2015-10-07T15:31:57Z
 gopkg.in/goose.v1	git	495e6fa2ab89bc5ed2c8e1bbcbc4c9e4a3c97d37	2016-03-17T17:25:46Z

provider/common/disk.go

@@ -5,16 +5,16 @@ package common
 
 import (
 	jujuos "github.com/juju/utils/os"
-	"github.com/juju/utils/series"
+	jujuseries "github.com/juju/utils/series"
 )
 
 // MinRootDiskSizeGiB is the minimum size for the root disk of an
 // instance, in Gigabytes. This value accommodates the anticipated
 // size of the initial image, any updates, and future application
 // data.
-func MinRootDiskSizeGiB(ser string) uint64 {
+func MinRootDiskSizeGiB(series string) uint64 {
 	// See comment below that explains why we're ignoring the error
-	os, _ := series.GetOSFromSeries(ser)
+	os, _ := jujuseries.GetOSFromSeries(series)
 	switch os {
 	case jujuos.Ubuntu, jujuos.CentOS:
 		return 8

provider/ec2/ebs.go

@@ -41,13 +41,18 @@ const (
 	EBS_Encrypted = "encrypted"
 
 	volumeTypeMagnetic        = "magnetic"         // standard
-	volumeTypeSsd             = "ssd"              // gp2
+	volumeTypeSSD             = "ssd"              // gp2
 	volumeTypeProvisionedIops = "provisioned-iops" // io1
 	volumeTypeStandard        = "standard"
-	volumeTypeGp2             = "gp2"
-	volumeTypeIo1             = "io1"
+	volumeTypeGP2             = "gp2"
+	volumeTypeIO1             = "io1"
 
 	rootDiskDeviceName = "/dev/sda1"
+
+	// defaultControllerDiskSizeMiB is the default size for the
+	// root disk of controller machines, if no root-disk constraint
+	// is specified.
+	defaultControllerDiskSizeMiB = 32 * 1024
 )
 
 // AWS error codes
@@ -78,11 +83,11 @@ const (
 	// maxMagneticVolumeSizeGiB is the maximum size for magnetic volumes in GiB.
 	maxMagneticVolumeSizeGiB = 1024
 
-	// minSsdVolumeSizeGiB is the minimum size for SSD volumes in GiB.
-	minSsdVolumeSizeGiB = 1
+	// minSSDVolumeSizeGiB is the minimum size for SSD volumes in GiB.
+	minSSDVolumeSizeGiB = 1
 
-	// maxSsdVolumeSizeGiB is the maximum size for SSD volumes in GiB.
-	maxSsdVolumeSizeGiB = 16 * 1024
+	// maxSSDVolumeSizeGiB is the maximum size for SSD volumes in GiB.
+	maxSSDVolumeSizeGiB = 16 * 1024
 
 	// minProvisionedIopsVolumeSizeGiB is the minimum size of provisioned IOPS
 	// volumes in GiB.
@@ -137,11 +142,11 @@ var _ storage.Provider = (*ebsProvider)(nil)
 var ebsConfigFields = schema.Fields{
 	EBS_VolumeType: schema.OneOf(
 		schema.Const(volumeTypeMagnetic),
-		schema.Const(volumeTypeSsd),
+		schema.Const(volumeTypeSSD),
 		schema.Const(volumeTypeProvisionedIops),
 		schema.Const(volumeTypeStandard),
-		schema.Const(volumeTypeGp2),
-		schema.Const(volumeTypeIo1),
+		schema.Const(volumeTypeGP2),
+		schema.Const(volumeTypeIO1),
 	),
 	EBS_IOPS:      schema.ForceInt(),
 	EBS_Encrypted: schema.Bool(),
@@ -178,15 +183,15 @@ func newEbsConfig(attrs map[string]interface{}) (*ebsConfig, error) {
 	switch ebsConfig.volumeType {
 	case volumeTypeMagnetic:
 		ebsConfig.volumeType = volumeTypeStandard
-	case volumeTypeSsd:
-		ebsConfig.volumeType = volumeTypeGp2
+	case volumeTypeSSD:
+		ebsConfig.volumeType = volumeTypeGP2
 	case volumeTypeProvisionedIops:
-		ebsConfig.volumeType = volumeTypeIo1
+		ebsConfig.volumeType = volumeTypeIO1
 	}
-	if ebsConfig.iops > 0 && ebsConfig.volumeType != volumeTypeIo1 {
+	if ebsConfig.iops > 0 && ebsConfig.volumeType != volumeTypeIO1 {
 		return nil, errors.Errorf("IOPS specified, but volume type is %q", volumeType)
-	} else if ebsConfig.iops == 0 && ebsConfig.volumeType == volumeTypeIo1 {
-		return nil, errors.Errorf("volume type is %q, IOPS unspecified or zero", volumeTypeIo1)
+	} else if ebsConfig.iops == 0 && ebsConfig.volumeType == volumeTypeIO1 {
+		return nil, errors.Errorf("volume type is %q, IOPS unspecified or zero", volumeTypeIO1)
 	}
 	return ebsConfig, nil
 }
@@ -215,7 +220,7 @@ func (e *ebsProvider) Dynamic() bool {
 // DefaultPools is defined on the Provider interface.
 func (e *ebsProvider) DefaultPools() []*storage.Config {
 	ssdPool, _ := storage.NewConfig("ebs-ssd", EBS_ProviderType, map[string]interface{}{
-		EBS_VolumeType: volumeTypeSsd,
+		EBS_VolumeType: volumeTypeSSD,
 	})
 	return []*storage.Config{ssdPool}
 }
@@ -579,10 +584,10 @@ func (v *ebsVolumeSource) ValidateVolumeParams(params storage.VolumeParams) erro
 	case volumeTypeStandard:
 		minVolumeSize = minMagneticVolumeSizeGiB
 		maxVolumeSize = maxMagneticVolumeSizeGiB
-	case volumeTypeGp2:
-		minVolumeSize = minSsdVolumeSizeGiB
-		maxVolumeSize = maxSsdVolumeSizeGiB
-	case volumeTypeIo1:
+	case volumeTypeGP2:
+		minVolumeSize = minSSDVolumeSizeGiB
+		maxVolumeSize = maxSSDVolumeSizeGiB
+	case volumeTypeIO1:
 		minVolumeSize = minProvisionedIopsVolumeSizeGiB
 		maxVolumeSize = maxProvisionedIopsVolumeSizeGiB
 	}
@@ -869,24 +874,32 @@ func blockDeviceNamer(numbers bool) func() (requestName, actualName string, err
 	}
 }
 
-func minRootDiskSizeMiB(ser string) uint64 {
-	return gibToMib(common.MinRootDiskSizeGiB(ser))
+func minRootDiskSizeMiB(series string) uint64 {
+	return gibToMib(common.MinRootDiskSizeGiB(series))
 }
 
 // getBlockDeviceMappings translates constraints into BlockDeviceMappings.
 //
 // The first entry is always the root disk mapping, followed by instance
 // stores (ephemeral disks).
-func getBlockDeviceMappings(cons constraints.Value, ser string) []ec2.BlockDeviceMapping {
-	rootDiskSizeMiB := minRootDiskSizeMiB(ser)
+func getBlockDeviceMappings(
+	cons constraints.Value,
+	series string,
+	controller bool,
+) []ec2.BlockDeviceMapping {
+	minRootDiskSizeMiB := minRootDiskSizeMiB(series)
+	rootDiskSizeMiB := minRootDiskSizeMiB
+	if controller {
+		rootDiskSizeMiB = defaultControllerDiskSizeMiB
+	}
 	if cons.RootDisk != nil {
-		if *cons.RootDisk >= minRootDiskSizeMiB(ser) {
+		if *cons.RootDisk >= minRootDiskSizeMiB {
 			rootDiskSizeMiB = *cons.RootDisk
 		} else {
 			logger.Infof(
-				"Ignoring root-disk constraint of %dM because it is smaller than the EC2 image size of %dM",
+				"Ignoring root-disk constraint of %dM because it is smaller than the minimum size %dM",
 				*cons.RootDisk,
-				minRootDiskSizeMiB(ser),
+				minRootDiskSizeMiB,
 			)
 		}
 	}

provider/ec2/ebs_test.go

@@ -728,7 +728,7 @@ func (*blockDeviceMappingSuite) TestBlockDeviceNamer(c *gc.C) {
 }
 
 func (*blockDeviceMappingSuite) TestGetBlockDeviceMappings(c *gc.C) {
-	mapping := ec2.GetBlockDeviceMappings(constraints.Value{}, "trusty")
+	mapping := ec2.GetBlockDeviceMappings(constraints.Value{}, "trusty", false)
 	c.Assert(mapping, gc.DeepEquals, []awsec2.BlockDeviceMapping{{
 		VolumeSize: 8,
 		DeviceName: "/dev/sda1",
@@ -746,3 +746,23 @@ func (*blockDeviceMappingSuite) TestGetBlockDeviceMappings(c *gc.C) {
 		DeviceName:  "/dev/sde",
 	}})
 }
+
+func (*blockDeviceMappingSuite) TestGetBlockDeviceMappingsController(c *gc.C) {
+	mapping := ec2.GetBlockDeviceMappings(constraints.Value{}, "trusty", true)
+	c.Assert(mapping, gc.DeepEquals, []awsec2.BlockDeviceMapping{{
+		VolumeSize: 32,
+		DeviceName: "/dev/sda1",
+	}, {
+		VirtualName: "ephemeral0",
+		DeviceName:  "/dev/sdb",
+	}, {
+		VirtualName: "ephemeral1",
+		DeviceName:  "/dev/sdc",
+	}, {
+		VirtualName: "ephemeral2",
+		DeviceName:  "/dev/sdd",
+	}, {
+		VirtualName: "ephemeral3",
+		DeviceName:  "/dev/sde",
+	}})
+}

provider/ec2/environ.go

@@ -66,6 +66,10 @@ type environ struct {
 
 	availabilityZonesMutex sync.Mutex
 	availabilityZones      []common.AvailabilityZone
+
+	defaultVPCMutex   sync.Mutex
+	defaultVPCChecked bool
+	defaultVPC        *ec2.VPC
 }
 
 func (e *environ) Config() *config.Config {
@@ -123,10 +127,6 @@ func (env *environ) Create(args environs.CreateParams) error {
 	return nil
 }
 
-func (env *environ) validateVPC(logInfof func(string, ...interface{}), badge string) error {
-	return nil
-}
-
 // Bootstrap is part of the Environ interface.
 func (e *environ) Bootstrap(ctx environs.BootstrapContext, args environs.BootstrapParams) (*environs.BootstrapResult, error) {
 	return common.Bootstrap(ctx, e, args)
@@ -156,7 +156,10 @@ func (e *environ) ConstraintsValidator() (constraints.Validator, error) {
 		[]string{constraints.InstanceType},
 		[]string{constraints.Mem, constraints.Cores, constraints.CpuPower})
 	validator.RegisterUnsupported(unsupportedConstraints)
-	instanceTypes := ec2instancetypes.RegionInstanceTypes(e.cloud.Region)
+	instanceTypes, err := e.supportedInstanceTypes()
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
 	instTypeNames := make([]string, len(instanceTypes))
 	for i, itype := range instanceTypes {
 		instTypeNames[i] = itype.Name
@@ -268,7 +271,11 @@ func (e *environ) PrecheckInstance(series string, cons constraints.Value, placem
 		return nil
 	}
 	// Constraint has an instance-type constraint so let's see if it is valid.
-	for _, itype := range ec2instancetypes.RegionInstanceTypes(e.cloud.Region) {
+	instanceTypes, err := e.supportedInstanceTypes()
+	if err != nil {
+		return errors.Trace(err)
+	}
+	for _, itype := range instanceTypes {
 		if itype.Name != *cons.InstanceType {
 			continue
 		}
@@ -390,13 +397,23 @@ func (e *environ) StartInstance(args environs.StartInstanceParams) (_ *environs.
 
 	arches := args.Tools.Arches()
 
-	spec, err := findInstanceSpec(args.ImageMetadata, &instances.InstanceConstraint{
-		Region:      e.cloud.Region,
-		Series:      args.InstanceConfig.Series,
-		Arches:      arches,
-		Constraints: args.Constraints,
-		Storage:     []string{ssdStorage, ebsStorage},
-	})
+	instanceTypes, err := e.supportedInstanceTypes()
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+
+	spec, err := findInstanceSpec(
+		args.InstanceConfig.Controller != nil,
+		args.ImageMetadata,
+		instanceTypes,
+		&instances.InstanceConstraint{
+			Region:      e.cloud.Region,
+			Series:      args.InstanceConfig.Series,
+			Arches:      arches,
+			Constraints: args.Constraints,
+			Storage:     []string{ssdStorage, ebsStorage},
+		},
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -432,7 +449,11 @@ func (e *environ) StartInstance(args environs.StartInstanceParams) (_ *environs.
 		return nil, errors.Annotate(err, "cannot set up groups")
 	}
 
-	blockDeviceMappings := getBlockDeviceMappings(args.Constraints, args.InstanceConfig.Series)
+	blockDeviceMappings := getBlockDeviceMappings(
+		args.Constraints,
+		args.InstanceConfig.Series,
+		args.InstanceConfig.Controller != nil,
+	)
 	rootDiskSize := uint64(blockDeviceMappings[0].VolumeSize) * 1024
 
 	// If --constraints spaces=foo was passed, the provisioner will populate
@@ -1695,3 +1716,47 @@ func (e *environ) AllocateContainerAddresses(hostInstanceID instance.Id, contain
 func (e *environ) ReleaseContainerAddresses(interfaces []network.ProviderInterfaceInfo) error {
 	return errors.NotSupportedf("container address allocation")
 }
+
+func (e *environ) supportedInstanceTypes() ([]instances.InstanceType, error) {
+	allInstanceTypes := ec2instancetypes.RegionInstanceTypes(e.cloud.Region)
+	if isVPCIDSet(e.ecfg().vpcID()) {
+		return allInstanceTypes, nil
+	}
+	hasDefaultVPC, err := e.hasDefaultVPC()
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	if hasDefaultVPC {
+		return allInstanceTypes, nil
+	}
+
+	// The region has no default VPC, and the user has not specified
+	// one to use. We filter out any instance types that are not
+	// supported in EC2-Classic.
+	supportedInstanceTypes := make([]instances.InstanceType, 0, len(allInstanceTypes))
+	for _, instanceType := range allInstanceTypes {
+		if !ec2instancetypes.SupportsClassic(instanceType.Name) {
+			continue
+		}
+		supportedInstanceTypes = append(supportedInstanceTypes, instanceType)
+	}
+	return supportedInstanceTypes, nil
+}
+
+func (e *environ) hasDefaultVPC() (bool, error) {
+	e.defaultVPCMutex.Lock()
+	defer e.defaultVPCMutex.Unlock()
+	if !e.defaultVPCChecked {
+		filter := ec2.NewFilter()
+		filter.Add("isDefault", "true")
+		resp, err := e.ec2.VPCs(nil, filter)
+		if err != nil {
+			return false, errors.Trace(err)
+		}
+		if len(resp.VPCs) > 0 {
+			e.defaultVPC = &resp.VPCs[0]
+		}
+		e.defaultVPCChecked = true
+	}
+	return e.defaultVPC != nil, nil
+}

provider/ec2/environ_test.go

@@ -96,7 +96,7 @@ func (*Suite) TestRootDiskBlockDeviceMapping(c *gc.C) {
 	for _, t := range rootDiskTests {
 		c.Logf("Test %s", t.name)
 		cons := constraints.Value{RootDisk: t.constraint}
-		mappings := getBlockDeviceMappings(cons, t.series)
+		mappings := getBlockDeviceMappings(cons, t.series, false)
 		expected := append([]amzec2.BlockDeviceMapping{t.device}, commonInstanceStoreDisks...)
 		c.Assert(mappings, gc.DeepEquals, expected)
 	}