diff --git a/api/v1alpha3/tags.go b/api/v1alpha3/tags.go index 191d8a3de27..2dbebddca3a 100644 --- a/api/v1alpha3/tags.go +++ b/api/v1alpha3/tags.go @@ -104,20 +104,23 @@ const ( // dedicated to this cluster api provider implementation. NameAzureClusterAPIRole = NameAzureProviderPrefix + "role" - // APIServerRoleTagValue describes the value for the apiserver role - APIServerRoleTagValue = "apiserver" + // APIServerRole describes the value for the apiserver role + APIServerRole = "apiserver" - // BastionRoleTagValue describes the value for the bastion role - BastionRoleTagValue = "bastion" + // NodeOutboundRole describes the value for the node outbound LB role + NodeOutboundRole = "nodeOutbound" - // CommonRoleTagValue describes the value for the common role - CommonRoleTagValue = "common" + // BastionRole describes the value for the bastion role + BastionRole = "bastion" - // PublicRoleTagValue describes the value for the public role - PublicRoleTagValue = "public" + // CommonRole describes the value for the common role + CommonRole = "common" - // PrivateRoleTagValue describes the value for the private role - PrivateRoleTagValue = "private" + // PublicRole describes the value for the public role + PublicRole = "public" + + // PrivateRole describes the value for the private role + PrivateRole = "private" ) // ClusterTagKey generates the key for resources associated with a cluster. diff --git a/cloud/defaults.go b/cloud/defaults.go index 8b6d10061f9..ea8ac8277bc 100644 --- a/cloud/defaults.go +++ b/cloud/defaults.go @@ -79,6 +79,11 @@ func GeneratePublicIPName(clusterName, hash string) string { return fmt.Sprintf("%s-%s", clusterName, hash) } +// GenerateNodeOutboundIPName generates a public IP name, based on the cluster name. +func GenerateNodeOutboundIPName(clusterName string) string { + return fmt.Sprintf("pip-%s-node-outbound", clusterName) +} + // GenerateNodePublicIPName generates a node public IP name, based on the NIC name. func GenerateNodePublicIPName(nicName string) string { return fmt.Sprintf("%s-public-ip", nicName) diff --git a/cloud/services/groups/groups.go b/cloud/services/groups/groups.go index 61de7923d50..466d04b7bde 100644 --- a/cloud/services/groups/groups.go +++ b/cloud/services/groups/groups.go @@ -51,7 +51,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { ClusterName: s.Scope.Name(), Lifecycle: infrav1.ResourceLifecycleOwned, Name: to.StringPtr(s.Scope.ResourceGroup()), - Role: to.StringPtr(infrav1.CommonRoleTagValue), + Role: to.StringPtr(infrav1.CommonRole), Additional: s.Scope.AdditionalTags(), })), } diff --git a/cloud/services/networkinterfaces/networkinterfaces.go b/cloud/services/networkinterfaces/networkinterfaces.go index a527a0d28ae..9fb4125bf9c 100644 --- a/cloud/services/networkinterfaces/networkinterfaces.go +++ b/cloud/services/networkinterfaces/networkinterfaces.go @@ -25,12 +25,14 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" "k8s.io/klog" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" ) // Spec specification for routetable type Spec struct { Name string + MachineRole string SubnetName string VnetName string StaticIPAddress string @@ -77,10 +79,9 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { backendAddressPools := []network.BackendAddressPool{} if nicSpec.PublicLoadBalancerName != "" { - // only control planes have an attached public LB lb, lberr := s.PublicLoadBalancersClient.Get(ctx, s.Scope.ResourceGroup(), nicSpec.PublicLoadBalancerName) if lberr != nil { - return errors.Wrap(lberr, "failed to get publicLB") + return errors.Wrap(lberr, "failed to get public LB") } backendAddressPools = append(backendAddressPools, @@ -88,16 +89,18 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { ID: (*lb.BackendAddressPools)[0].ID, }) - ruleName := s.MachineScope.Name() - naterr := s.createInboundNatRule(ctx, lb, ruleName) - if naterr != nil { - return errors.Wrap(naterr, "failed to create NAT rule") - } + if nicSpec.MachineRole == infrav1.ControlPlane { + ruleName := s.MachineScope.Name() + naterr := s.createInboundNatRule(ctx, lb, ruleName) + if naterr != nil { + return errors.Wrap(naterr, "failed to create NAT rule") + } - nicConfig.LoadBalancerInboundNatRules = &[]network.InboundNatRule{ - { - ID: to.StringPtr(fmt.Sprintf("%s/inboundNatRules/%s", to.String(lb.ID), ruleName)), - }, + nicConfig.LoadBalancerInboundNatRules = &[]network.InboundNatRule{ + { + ID: to.StringPtr(fmt.Sprintf("%s/inboundNatRules/%s", to.String(lb.ID), ruleName)), + }, + } } } if nicSpec.InternalLoadBalancerName != "" { diff --git a/cloud/services/networkinterfaces/networkinterfaces_test.go b/cloud/services/networkinterfaces/networkinterfaces_test.go index f671eeb0455..81e558b4345 100644 --- a/cloud/services/networkinterfaces/networkinterfaces_test.go +++ b/cloud/services/networkinterfaces/networkinterfaces_test.go @@ -244,9 +244,11 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "get subnets fails", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "failed to get subnets: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -263,9 +265,11 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "node network interface create fails", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "failed to create network interface my-net-interface in resource group my-rg: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -279,6 +283,7 @@ func TestReconcileNetworkInterface(t *testing.T) { gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet"). Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{})). Return(autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))) }, @@ -286,10 +291,12 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "node network interface with Static private IP successfully created", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", - StaticIPAddress: "1.2.3.4", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + StaticIPAddress: "1.2.3.4", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -302,15 +309,18 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku.EXPECT().HasAcceleratedNetworking(gomock.Any(), gomock.Any()) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{}))) }, }, { name: "node network interface with Dynamic private IP successfully created", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -323,6 +333,7 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku.EXPECT().HasAcceleratedNetworking(gomock.Any(), gomock.Any()) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{}))) }, }, @@ -334,6 +345,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", PublicLoadBalancerName: "my-publiclb", InternalLoadBalancerName: "my-internal-lb", + MachineRole: infrav1.ControlPlane, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -413,8 +425,9 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", PublicLoadBalancerName: "my-publiclb", InternalLoadBalancerName: "my-internal-lb", + MachineRole: infrav1.ControlPlane, }, - expectedError: "failed to get publicLB: #: Internal Server Error: StatusCode=500", + expectedError: "failed to get public LB: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, mSubnet *mock_subnets.MockClientMockRecorder, mPublicLoadBalancer *mock_publicloadbalancers.MockClientMockRecorder, @@ -436,6 +449,7 @@ func TestReconcileNetworkInterface(t *testing.T) { SubnetName: "my-subnet", PublicLoadBalancerName: "my-publiclb", InternalLoadBalancerName: "my-internal-lb", + MachineRole: infrav1.ControlPlane, }, expectedError: "failed to create NAT rule: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -501,6 +515,7 @@ func TestReconcileNetworkInterface(t *testing.T) { VnetName: "my-vnet", SubnetName: "my-subnet", InternalLoadBalancerName: "my-internal-lb", + MachineRole: infrav1.ControlPlane, }, expectedError: "failed to get internalLB: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -519,10 +534,12 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "network interface with Public IP successfully created", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", - PublicIPName: "my-public-ip", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicIPName: "my-public-ip", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -535,6 +552,7 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku.EXPECT().HasAcceleratedNetworking(gomock.Any(), gomock.Any()) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), mPublicIP.CreateOrUpdate(context.TODO(), "my-rg", "my-public-ip", gomock.AssignableToTypeOf(network.PublicIPAddress{})), mPublicIP.Get(context.TODO(), "my-rg", "my-public-ip").Return(network.PublicIPAddress{}, nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{}))) @@ -543,10 +561,12 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "network interface with Public IP fail to get Public IP", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", - PublicIPName: "my-public-ip", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicIPName: "my-public-ip", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "failed to get publicIP: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -558,17 +578,19 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku *mock_resourceskus.MockClient) { gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), mPublicIP.CreateOrUpdate(context.TODO(), "my-rg", "my-public-ip", gomock.AssignableToTypeOf(network.PublicIPAddress{})), - mPublicIP.Get(context.TODO(), "my-rg", "my-public-ip").Return(network.PublicIPAddress{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")), - m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{}))) + mPublicIP.Get(context.TODO(), "my-rg", "my-public-ip").Return(network.PublicIPAddress{}, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error"))) }, }, { name: "network interface with accelerated networking successfully created", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -581,6 +603,7 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku.EXPECT().HasAcceleratedNetworking(context.TODO(), gomock.Any()).Return(true, nil) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", matchers.DiffEq(network.Interface{ Location: to.StringPtr("test-location"), InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ @@ -591,7 +614,7 @@ func TestReconcileNetworkInterface(t *testing.T) { InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &network.Subnet{}, PrivateIPAllocationMethod: network.Dynamic, - LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{{ID: to.StringPtr("cluster-name-outboundBackendPool")}}, }, }, }, @@ -603,9 +626,11 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "network interface without accelerated networking successfully created", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -618,6 +643,7 @@ func TestReconcileNetworkInterface(t *testing.T) { mResourceSku.EXPECT().HasAcceleratedNetworking(context.TODO(), gomock.Any()).Return(false, nil) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", matchers.DiffEq(network.Interface{ Location: to.StringPtr("test-location"), InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ @@ -628,7 +654,7 @@ func TestReconcileNetworkInterface(t *testing.T) { InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &network.Subnet{}, PrivateIPAllocationMethod: network.Dynamic, - LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{}, + LoadBalancerBackendAddressPools: &[]network.BackendAddressPool{{ID: to.StringPtr("cluster-name-outboundBackendPool")}}, }, }, }, @@ -640,9 +666,11 @@ func TestReconcileNetworkInterface(t *testing.T) { { name: "network interface fails to get accelerated networking capability", netInterfaceSpec: Spec{ - Name: "my-net-interface", - VnetName: "my-vnet", - SubnetName: "my-subnet", + Name: "my-net-interface", + VnetName: "my-vnet", + SubnetName: "my-subnet", + PublicLoadBalancerName: "my-cluster", + MachineRole: infrav1.Node, }, expectedError: "failed to get accelerated networking capability: #: Internal Server Error: StatusCode=500", expect: func(m *mock_networkinterfaces.MockClientMockRecorder, @@ -656,6 +684,7 @@ func TestReconcileNetworkInterface(t *testing.T) { false, autorest.NewErrorWithResponse("", "", &http.Response{StatusCode: 500}, "Internal Server Error")) gomock.InOrder( mSubnet.Get(context.TODO(), "my-rg", "my-vnet", "my-subnet").Return(network.Subnet{}, nil), + mPublicLoadBalancer.Get(context.TODO(), "my-rg", "my-cluster").Return(getFakeNodeOutboundLoadBalancer(), nil), m.CreateOrUpdate(context.TODO(), "my-rg", "my-net-interface", gomock.AssignableToTypeOf(network.Interface{})), ) }, @@ -953,3 +982,19 @@ func TestDeleteNetworkInterface(t *testing.T) { }) } } + +func getFakeNodeOutboundLoadBalancer() network.LoadBalancer { + return network.LoadBalancer{ + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + ID: to.StringPtr("frontend-ip-config-id"), + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + { + ID: pointer.StringPtr("cluster-name-outboundBackendPool"), + }, + }, + }} +} diff --git a/cloud/services/publicloadbalancers/publicloadbalancers.go b/cloud/services/publicloadbalancers/publicloadbalancers.go index a935988f8ef..bedebe356a9 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers.go @@ -33,6 +33,7 @@ import ( type Spec struct { Name string PublicIPName string + Role string } // Get provides information about a public load balancer. @@ -56,11 +57,14 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { if !ok { return errors.New("invalid public loadbalancer specification") } - probeName := "tcpHTTPSProbe" - frontEndIPConfigName := "controlplane-lbFrontEnd" - backEndAddressPoolName := "controlplane-backEndPool" - idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.ResourceGroup()) lbName := publicLBSpec.Name + frontEndIPConfigName := fmt.Sprintf("%s-%s", publicLBSpec.Name, "frontEnd") + backEndAddressPoolName := fmt.Sprintf("%s-%s", publicLBSpec.Name, "backendPool") + if publicLBSpec.Role == infrav1.NodeOutboundRole { + backEndAddressPoolName = fmt.Sprintf("%s-%s", publicLBSpec.Name, "outboundBackendPool") + } + idPrefix := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/loadBalancers", s.Scope.SubscriptionID, s.Scope.ResourceGroup()) + klog.V(2).Infof("creating public load balancer %s", lbName) klog.V(2).Infof("getting public ip %s", publicLBSpec.PublicIPName) @@ -72,89 +76,91 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { } klog.V(2).Infof("successfully got public ip %s", publicLBSpec.PublicIPName) - // https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-standard-availability-zones#zone-redundant-by-default - err = s.Client.CreateOrUpdate(ctx, - s.Scope.ResourceGroup(), - lbName, - network.LoadBalancer{ - Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ - ClusterName: s.Scope.Name(), - Lifecycle: infrav1.ResourceLifecycleOwned, - Role: to.StringPtr(infrav1.APIServerRoleTagValue), - Additional: s.Scope.AdditionalTags(), - })), - Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, - Location: to.StringPtr(s.Scope.Location()), - LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ - FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ - { - Name: &frontEndIPConfigName, - FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.Dynamic, - PublicIPAddress: &publicIP, - }, - }, - }, - BackendAddressPools: &[]network.BackendAddressPool{ - { - Name: &backEndAddressPoolName, + lb := network.LoadBalancer{ + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Location: to.StringPtr(s.Scope.Location()), + Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ + ClusterName: s.Scope.Name(), + Lifecycle: infrav1.ResourceLifecycleOwned, + Role: to.StringPtr(publicLBSpec.Role), + Additional: s.Scope.AdditionalTags(), + })), + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: &frontEndIPConfigName, + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.Dynamic, + PublicIPAddress: &publicIP, }, }, - Probes: &[]network.Probe{ - { - Name: &probeName, - ProbePropertiesFormat: &network.ProbePropertiesFormat{ - Protocol: network.ProbeProtocolTCP, - Port: to.Int32Ptr(s.Scope.APIServerPort()), - IntervalInSeconds: to.Int32Ptr(15), - NumberOfProbes: to.Int32Ptr(4), - }, - }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + { + Name: &backEndAddressPoolName, }, - // We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule. - // For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections. - LoadBalancingRules: &[]network.LoadBalancingRule{ - { - Name: to.StringPtr("LBRuleHTTPS"), - LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ - DisableOutboundSnat: to.BoolPtr(true), - Protocol: network.TransportProtocolTCP, - FrontendPort: to.Int32Ptr(s.Scope.APIServerPort()), - BackendPort: to.Int32Ptr(s.Scope.APIServerPort()), - IdleTimeoutInMinutes: to.Int32Ptr(4), - EnableFloatingIP: to.BoolPtr(false), - LoadDistribution: network.LoadDistributionDefault, - FrontendIPConfiguration: &network.SubResource{ + }, + OutboundRules: &[]network.OutboundRule{ + { + Name: to.StringPtr("OutboundNATAllProtocols"), + OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + Protocol: network.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(4), + FrontendIPConfigurations: &[]network.SubResource{ + { ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), - }, - Probe: &network.SubResource{ - ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), - }, + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), }, }, }, - OutboundRules: &[]network.OutboundRule{ - { - Name: to.StringPtr("OutboundNATAllProtocols"), - OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ - Protocol: network.LoadBalancerOutboundRuleProtocolAll, - IdleTimeoutInMinutes: to.Int32Ptr(4), - FrontendIPConfigurations: &[]network.SubResource{ - { - ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), - }, - }, - BackendAddressPool: &network.SubResource{ - ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), - }, - }, + }, + }, + } + + if publicLBSpec.Role == infrav1.APIServerRole { + probeName := "tcpHTTPSProbe" + lb.LoadBalancerPropertiesFormat.Probes = &[]network.Probe{ + { + Name: to.StringPtr(probeName), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: network.ProbeProtocolTCP, + Port: to.Int32Ptr(s.Scope.APIServerPort()), + IntervalInSeconds: to.Int32Ptr(15), + NumberOfProbes: to.Int32Ptr(4), + }, + }, + } + // We disable outbound SNAT explicitly in the HTTPS LB rule and enable TCP and UDP outbound NAT with an outbound rule. + // For more information on Standard LB outbound connections see https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-outbound-connections. + lb.LoadBalancerPropertiesFormat.LoadBalancingRules = &[]network.LoadBalancingRule{ + { + Name: to.StringPtr("LBRuleHTTPS"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + DisableOutboundSnat: to.BoolPtr(true), + Protocol: network.TransportProtocolTCP, + FrontendPort: to.Int32Ptr(s.Scope.APIServerPort()), + BackendPort: to.Int32Ptr(s.Scope.APIServerPort()), + IdleTimeoutInMinutes: to.Int32Ptr(4), + EnableFloatingIP: to.BoolPtr(false), + LoadDistribution: network.LoadDistributionDefault, + FrontendIPConfiguration: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/frontendIPConfigurations/%s", idPrefix, lbName, frontEndIPConfigName)), + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/backendAddressPools/%s", idPrefix, lbName, backEndAddressPoolName)), + }, + Probe: &network.SubResource{ + ID: to.StringPtr(fmt.Sprintf("/%s/%s/probes/%s", idPrefix, lbName, probeName)), }, }, }, - }) + } + } + + err = s.Client.CreateOrUpdate(ctx, s.Scope.ResourceGroup(), lbName, lb) if err != nil { return errors.Wrap(err, "cannot create public load balancer") diff --git a/cloud/services/publicloadbalancers/publicloadbalancers_test.go b/cloud/services/publicloadbalancers/publicloadbalancers_test.go index 58afed2de80..992dac9b664 100644 --- a/cloud/services/publicloadbalancers/publicloadbalancers_test.go +++ b/cloud/services/publicloadbalancers/publicloadbalancers_test.go @@ -24,8 +24,10 @@ import ( . "github.com/onsi/gomega" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicips/mock_publicips" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" + "sigs.k8s.io/cluster-api-provider-azure/internal/test/matchers" "github.com/Azure/go-autorest/autorest" + "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" network "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" @@ -249,6 +251,146 @@ func TestReconcilePublicLoadBalancer(t *testing.T) { publicIP.Get(context.TODO(), "my-rg", "my-publicip").Return(network.PublicIPAddress{}, nil) }, }, + { + name: "create apiserver LB", + publicLBSpec: Spec{ + Name: "my-publiclb", + PublicIPName: "my-publicip", + Role: infrav1.APIServerRole, + }, + expectedError: "", + expect: func(m *mock_publicloadbalancers.MockClientMockRecorder, + publicIP *mock_publicips.MockClientMockRecorder) { + gomock.InOrder( + publicIP.Get(context.TODO(), "my-rg", "my-publicip").Return(network.PublicIPAddress{Name: to.StringPtr("my-publicip")}, nil), + m.CreateOrUpdate(context.TODO(), "my-rg", "my-publiclb", matchers.DiffEq(network.LoadBalancer{ + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": to.StringPtr("owned"), + "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.APIServerRole), + }, + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Location: to.StringPtr("test-location"), + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: to.StringPtr("my-publiclb-frontEnd"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.Dynamic, + PublicIPAddress: &network.PublicIPAddress{Name: to.StringPtr("my-publicip")}, + }, + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + { + Name: to.StringPtr("my-publiclb-backendPool"), + }, + }, + LoadBalancingRules: &[]network.LoadBalancingRule{ + { + Name: to.StringPtr("LBRuleHTTPS"), + LoadBalancingRulePropertiesFormat: &network.LoadBalancingRulePropertiesFormat{ + DisableOutboundSnat: to.BoolPtr(true), + Protocol: network.TransportProtocolTCP, + FrontendPort: to.Int32Ptr(6443), + BackendPort: to.Int32Ptr(6443), + IdleTimeoutInMinutes: to.Int32Ptr(4), + EnableFloatingIP: to.BoolPtr(false), + LoadDistribution: network.LoadDistributionDefault, + FrontendIPConfiguration: &network.SubResource{ + ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd"), + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/backendAddressPools/my-publiclb-backendPool"), + }, + Probe: &network.SubResource{ + ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/probes/tcpHTTPSProbe"), + }, + }, + }, + }, + Probes: &[]network.Probe{ + { + Name: to.StringPtr("tcpHTTPSProbe"), + ProbePropertiesFormat: &network.ProbePropertiesFormat{ + Protocol: network.ProbeProtocolTCP, + Port: to.Int32Ptr(6443), + IntervalInSeconds: to.Int32Ptr(15), + NumberOfProbes: to.Int32Ptr(4), + }, + }, + }, + OutboundRules: &[]network.OutboundRule{ + { + Name: to.StringPtr("OutboundNATAllProtocols"), + OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]network.SubResource{ + {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/frontendIPConfigurations/my-publiclb-frontEnd")}, + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/my-publiclb/backendAddressPools/my-publiclb-backendPool"), + }, + Protocol: network.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(4), + }, + }, + }, + }, + })).Return(nil)) + }, + }, + { + name: "create node outbound LB", + publicLBSpec: Spec{ + Name: "cluster-name", + PublicIPName: "outbound-publicip", + Role: infrav1.NodeOutboundRole, + }, + expectedError: "", + expect: func(m *mock_publicloadbalancers.MockClientMockRecorder, + publicIP *mock_publicips.MockClientMockRecorder) { + gomock.InOrder( + publicIP.Get(context.TODO(), "my-rg", "outbound-publicip").Return(network.PublicIPAddress{Name: to.StringPtr("outbound-publicip")}, nil), + m.CreateOrUpdate(context.TODO(), "my-rg", "cluster-name", matchers.DiffEq(network.LoadBalancer{ + Tags: map[string]*string{ + "sigs.k8s.io_cluster-api-provider-azure_cluster_test-cluster": to.StringPtr("owned"), + "sigs.k8s.io_cluster-api-provider-azure_role": to.StringPtr(infrav1.NodeOutboundRole), + }, + Sku: &network.LoadBalancerSku{Name: network.LoadBalancerSkuNameStandard}, + Location: to.StringPtr("test-location"), + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + Name: to.StringPtr("cluster-name-frontEnd"), + FrontendIPConfigurationPropertiesFormat: &network.FrontendIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.Dynamic, + PublicIPAddress: &network.PublicIPAddress{Name: to.StringPtr("outbound-publicip")}, + }, + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + { + Name: to.StringPtr("cluster-name-outboundBackendPool"), + }, + }, + OutboundRules: &[]network.OutboundRule{ + { + Name: to.StringPtr("OutboundNATAllProtocols"), + OutboundRulePropertiesFormat: &network.OutboundRulePropertiesFormat{ + FrontendIPConfigurations: &[]network.SubResource{ + {ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/frontendIPConfigurations/cluster-name-frontEnd")}, + }, + BackendAddressPool: &network.SubResource{ + ID: to.StringPtr("//subscriptions/123/resourceGroups/my-rg/providers/Microsoft.Network/loadBalancers/cluster-name/backendAddressPools/cluster-name-outboundBackendPool"), + }, + Protocol: network.LoadBalancerOutboundRuleProtocolAll, + IdleTimeoutInMinutes: to.Int32Ptr(4), + }, + }, + }, + }, + })).Return(nil)) + }, + }, } for _, tc := range testcases { diff --git a/cloud/services/scalesets/service.go b/cloud/services/scalesets/service.go index c7e5f8e3cd1..3914611b880 100644 --- a/cloud/services/scalesets/service.go +++ b/cloud/services/scalesets/service.go @@ -17,19 +17,22 @@ package scalesets import ( "github.com/Azure/go-autorest/autorest" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus" ) // Service provides operations on azure resources type Service struct { Client - ResourceSkusClient resourceskus.Client + ResourceSkusClient resourceskus.Client + PublicLoadBalancersClient publicloadbalancers.Client } // NewService creates a new service. func NewService(authorizer autorest.Authorizer, subscriptionID string) *Service { return &Service{ - Client: NewClient(subscriptionID, authorizer), - ResourceSkusClient: resourceskus.NewClient(subscriptionID, authorizer), + Client: NewClient(subscriptionID, authorizer), + ResourceSkusClient: resourceskus.NewClient(subscriptionID, authorizer), + PublicLoadBalancersClient: publicloadbalancers.NewClient(subscriptionID, authorizer), } } diff --git a/cloud/services/scalesets/vmss.go b/cloud/services/scalesets/vmss.go index 5509dd5b0da..c3d9cd2de62 100644 --- a/cloud/services/scalesets/vmss.go +++ b/cloud/services/scalesets/vmss.go @@ -34,20 +34,21 @@ import ( // Spec input specification for Get/CreateOrUpdate/Delete calls type ( Spec struct { - Name string - ResourceGroup string - Location string - ClusterName string - MachinePoolName string - Sku string - Capacity int64 - SSHKeyData string - Image *infrav1.Image - OSDisk infrav1.OSDisk - CustomData string - SubnetID string - AdditionalTags infrav1.Tags - AcceleratedNetworking *bool + Name string + ResourceGroup string + Location string + ClusterName string + MachinePoolName string + Sku string + Capacity int64 + SSHKeyData string + Image *infrav1.Image + OSDisk infrav1.OSDisk + CustomData string + SubnetID string + PublicLoadBalancerName string + AdditionalTags infrav1.Tags + AcceleratedNetworking *bool } ) @@ -97,6 +98,18 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { vmssSpec.AcceleratedNetworking = to.BoolPtr(accelNet) } + // Get the node outbound LB backend pool ID + lb, lberr := s.PublicLoadBalancersClient.Get(ctx, vmssSpec.ResourceGroup, vmssSpec.PublicLoadBalancerName) + if lberr != nil { + return errors.Wrap(lberr, "failed to get cloud provider LB") + } + + backendAddressPools := []compute.SubResource{ + { + ID: (*lb.BackendAddressPools)[0].ID, + }, + } + vmss := compute.VirtualMachineScaleSet{ Location: to.StringPtr(vmssSpec.Location), Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{ @@ -147,8 +160,9 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { Subnet: &compute.APIEntityReference{ ID: to.StringPtr(vmssSpec.SubnetID), }, - Primary: to.BoolPtr(true), - PrivateIPAddressVersion: compute.IPv4, + Primary: to.BoolPtr(true), + PrivateIPAddressVersion: compute.IPv4, + LoadBalancerBackendAddressPools: &backendAddressPools, }, }, }, diff --git a/cloud/services/scalesets/vmss_test.go b/cloud/services/scalesets/vmss_test.go index 70f5836a201..78f44da539e 100644 --- a/cloud/services/scalesets/vmss_test.go +++ b/cloud/services/scalesets/vmss_test.go @@ -21,13 +21,15 @@ import ( "fmt" "testing" - "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-06-01/network" "github.com/Azure/go-autorest/autorest" "github.com/Azure/go-autorest/autorest/to" "github.com/golang/mock/gomock" "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -35,6 +37,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" + "sigs.k8s.io/cluster-api-provider-azure/cloud/services/publicloadbalancers/mock_publicloadbalancers" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/resourceskus/mock_resourceskus" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/scalesets/mock_scalesets" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" @@ -104,12 +107,13 @@ func TestService_Get(t *testing.T) { Name: "WithValidSpecBut404FromAzureOnVMSS", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), } }, Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) { @@ -130,12 +134,13 @@ func TestService_Get(t *testing.T) { Name: "WithValidSpecBut404FromAzureOnInstances", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), } }, Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) { @@ -157,12 +162,13 @@ func TestService_Get(t *testing.T) { Name: "WithValidSpecWithVMSSAndInstancesReturned", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), } }, Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) { @@ -247,15 +253,16 @@ func TestService_Reconcile(t *testing.T) { Name: "WithValidSpec", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), - Sku: "skuName", - Capacity: 2, - SSHKeyData: "sshKeyData", + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), + Sku: "skuName", + Capacity: 2, + SSHKeyData: "sshKeyData", OSDisk: infrav1.OSDisk{ OSType: "Linux", DiskSizeGB: 120, @@ -275,6 +282,8 @@ func TestService_Reconcile(t *testing.T) { svc.Client = vmssMock skusMock := mock_resourceskus.NewMockClient(mockCtrl) svc.ResourceSkusClient = skusMock + lbMock := mock_publicloadbalancers.NewMockClient(mockCtrl) + svc.PublicLoadBalancersClient = lbMock storageProfile, err := generateStorageProfile(*spec) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -329,8 +338,9 @@ func TestService_Reconcile(t *testing.T) { Subnet: &compute.APIEntityReference{ ID: to.StringPtr(scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID), }, - Primary: to.BoolPtr(true), - PrivateIPAddressVersion: compute.IPv4, + Primary: to.BoolPtr(true), + PrivateIPAddressVersion: compute.IPv4, + LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr("cluster-name-outboundBackendPool")}}, }, }, }, @@ -343,6 +353,7 @@ func TestService_Reconcile(t *testing.T) { } skusMock.EXPECT().HasAcceleratedNetworking(gomock.Any(), gomock.Any()).Return(false, nil) + lbMock.EXPECT().Get(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.ClusterName).Return(getFakeNodeOutboundLoadBalancer(), nil) vmssMock.EXPECT().CreateOrUpdate(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.Name, matchers.DiffEq(vmss)).Return(nil) }, Expect: func(ctx context.Context, g *gomega.GomegaWithT, err error) { @@ -353,15 +364,16 @@ func TestService_Reconcile(t *testing.T) { Name: "WithAcceleratedNetworking", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), - Sku: "skuName", - Capacity: 2, - SSHKeyData: "sshKeyData", + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), + Sku: "skuName", + Capacity: 2, + SSHKeyData: "sshKeyData", OSDisk: infrav1.OSDisk{ OSType: "Linux", DiskSizeGB: 120, @@ -381,6 +393,8 @@ func TestService_Reconcile(t *testing.T) { svc.Client = vmssMock skusMock := mock_resourceskus.NewMockClient(mockCtrl) svc.ResourceSkusClient = skusMock + lbMock := mock_publicloadbalancers.NewMockClient(mockCtrl) + svc.PublicLoadBalancersClient = lbMock storageProfile, err := generateStorageProfile(*spec) g.Expect(err).ToNot(gomega.HaveOccurred()) @@ -435,8 +449,9 @@ func TestService_Reconcile(t *testing.T) { Subnet: &compute.APIEntityReference{ ID: to.StringPtr(scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID), }, - Primary: to.BoolPtr(true), - PrivateIPAddressVersion: compute.IPv4, + Primary: to.BoolPtr(true), + PrivateIPAddressVersion: compute.IPv4, + LoadBalancerBackendAddressPools: &[]compute.SubResource{{ID: to.StringPtr("cluster-name-outboundBackendPool")}}, }, }, }, @@ -449,6 +464,7 @@ func TestService_Reconcile(t *testing.T) { } skusMock.EXPECT().HasAcceleratedNetworking(gomock.Any(), gomock.Any()).Return(true, nil) + lbMock.EXPECT().Get(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.ClusterName).Return(getFakeNodeOutboundLoadBalancer(), nil) vmssMock.EXPECT().CreateOrUpdate(gomock.Any(), scope.AzureCluster.Spec.ResourceGroup, spec.Name, matchers.DiffEq(vmss)).Return(nil) }, Expect: func(ctx context.Context, g *gomega.GomegaWithT, err error) { @@ -496,12 +512,13 @@ func TestService_Delete(t *testing.T) { Name: "WithValidSpecBut404FromAzureOnVMSSAssumeAlreadyDeleted", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), } }, Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) { @@ -521,12 +538,13 @@ func TestService_Delete(t *testing.T) { Name: "WithValidSpecAndSuccessfulDelete", SpecFactory: func(g *gomega.GomegaWithT, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) interface{} { return &Spec{ - Name: mpScope.Name(), - ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, - Location: scope.AzureCluster.Spec.Location, - ClusterName: scope.Cluster.Name, - SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - MachinePoolName: mpScope.Name(), + Name: mpScope.Name(), + ResourceGroup: scope.AzureCluster.Spec.ResourceGroup, + Location: scope.AzureCluster.Spec.Location, + ClusterName: scope.Cluster.Name, + SubnetID: scope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: scope.Cluster.Name, + MachinePoolName: mpScope.Name(), } }, Setup: func(ctx context.Context, g *gomega.GomegaWithT, svc *Service, scope *scope.ClusterScope, mpScope *scope.MachinePoolScope) { @@ -605,3 +623,19 @@ func getScopes(g *gomega.GomegaWithT) (*scope.ClusterScope, *scope.MachinePoolSc return s, mps } + +func getFakeNodeOutboundLoadBalancer() network.LoadBalancer { + return network.LoadBalancer{ + LoadBalancerPropertiesFormat: &network.LoadBalancerPropertiesFormat{ + FrontendIPConfigurations: &[]network.FrontendIPConfiguration{ + { + ID: to.StringPtr("frontend-ip-config-id"), + }, + }, + BackendAddressPools: &[]network.BackendAddressPool{ + { + ID: pointer.StringPtr("cluster-name-outboundBackendPool"), + }, + }, + }} +} diff --git a/cloud/services/virtualnetworks/virtualnetworks.go b/cloud/services/virtualnetworks/virtualnetworks.go index 87554a46eb3..3be1faab18b 100644 --- a/cloud/services/virtualnetworks/virtualnetworks.go +++ b/cloud/services/virtualnetworks/virtualnetworks.go @@ -98,7 +98,7 @@ func (s *Service) Reconcile(ctx context.Context, spec interface{}) error { ClusterName: s.Scope.Name(), Lifecycle: infrav1.ResourceLifecycleOwned, Name: to.StringPtr(vnetSpec.Name), - Role: to.StringPtr(infrav1.CommonRoleTagValue), + Role: to.StringPtr(infrav1.CommonRole), Additional: s.Scope.AdditionalTags(), })), Location: to.StringPtr(s.Scope.Location()), diff --git a/controllers/azurecluster_reconciler.go b/controllers/azurecluster_reconciler.go index c45e734b3b0..c55fed06ef6 100644 --- a/controllers/azurecluster_reconciler.go +++ b/controllers/azurecluster_reconciler.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3" azure "sigs.k8s.io/cluster-api-provider-azure/cloud" "sigs.k8s.io/cluster-api-provider-azure/cloud/scope" "sigs.k8s.io/cluster-api-provider-azure/cloud/services/availabilityzones" @@ -159,11 +160,28 @@ func (r *azureClusterReconciler) Reconcile() error { publicLBSpec := &publicloadbalancers.Spec{ Name: azure.GeneratePublicLBName(r.scope.Name()), PublicIPName: r.scope.Network().APIServerIP.Name, + Role: infrav1.APIServerRole, } if err := r.publicLBSvc.Reconcile(r.scope.Context, publicLBSpec); err != nil { return errors.Wrapf(err, "failed to reconcile control plane public load balancer for cluster %s", r.scope.Name()) } + nodeOutboundPublicIPSpec := &publicips.Spec{ + Name: azure.GenerateNodeOutboundIPName(r.scope.Name()), + } + if err := r.publicIPSvc.Reconcile(r.scope.Context, nodeOutboundPublicIPSpec); err != nil { + return errors.Wrapf(err, "failed to reconcile node outbound public ip for cluster %s", r.scope.Name()) + } + + nodeOutboundLBSpec := &publicloadbalancers.Spec{ + Name: r.scope.Name(), + PublicIPName: azure.GenerateNodeOutboundIPName(r.scope.Name()), + Role: infrav1.NodeOutboundRole, + } + if err := r.publicLBSvc.Reconcile(r.scope.Context, nodeOutboundLBSpec); err != nil { + return errors.Wrapf(err, "failed to reconcile node outbound public load balancer for cluster %s", r.scope.Name()) + } + return nil } @@ -215,7 +233,7 @@ func (r *azureClusterReconciler) deleteLB() error { } if err := r.publicLBSvc.Delete(r.scope.Context, publicLBSpec); err != nil { if !azure.ResourceNotFound(err) { - return errors.Wrapf(err, "failed to delete lb %s for cluster %s", azure.GeneratePublicLBName(r.scope.Name()), r.scope.Name()) + return errors.Wrapf(err, "failed to delete lb %s for cluster %s", publicLBSpec.Name, r.scope.Name()) } } publicIPSpec := &publicips.Spec{ @@ -223,7 +241,24 @@ func (r *azureClusterReconciler) deleteLB() error { } if err := r.publicIPSvc.Delete(r.scope.Context, publicIPSpec); err != nil { if !azure.ResourceNotFound(err) { - return errors.Wrapf(err, "failed to delete public ip %s for cluster %s", r.scope.Network().APIServerIP.Name, r.scope.Name()) + return errors.Wrapf(err, "failed to delete public ip %s for cluster %s", publicIPSpec.Name, r.scope.Name()) + } + } + + nodeOutboundLBSpec := &publicloadbalancers.Spec{ + Name: r.scope.Name(), + } + if err := r.publicLBSvc.Delete(r.scope.Context, nodeOutboundLBSpec); err != nil { + if !azure.ResourceNotFound(err) { + return errors.Wrapf(err, "failed to delete lb %s for cluster %s", nodeOutboundLBSpec.Name, r.scope.Name()) + } + } + nodeOutboundPublicIPSpec := &publicips.Spec{ + Name: azure.GenerateNodeOutboundIPName(r.scope.Name()), + } + if err := r.publicIPSvc.Delete(r.scope.Context, nodeOutboundPublicIPSpec); err != nil { + if !azure.ResourceNotFound(err) { + return errors.Wrapf(err, "failed to delete public ip %s for cluster %s", nodeOutboundPublicIPSpec.Name, r.scope.Name()) } } diff --git a/controllers/azuremachine_reconciler.go b/controllers/azuremachine_reconciler.go index 5e97d21cf36..fe193d8511d 100644 --- a/controllers/azuremachine_reconciler.go +++ b/controllers/azuremachine_reconciler.go @@ -89,12 +89,15 @@ func (s *azureMachineService) Delete() error { } networkInterfaceSpec := &networkinterfaces.Spec{ - Name: azure.GenerateNICName(s.machineScope.Name()), - VnetName: s.clusterScope.Vnet().Name, + Name: azure.GenerateNICName(s.machineScope.Name()), + VnetName: s.clusterScope.Vnet().Name, + MachineRole: s.machineScope.Role(), } if s.machineScope.Role() == infrav1.ControlPlane { networkInterfaceSpec.PublicLoadBalancerName = azure.GeneratePublicLBName(s.clusterScope.Name()) + } else if s.machineScope.Role() == infrav1.Node { + networkInterfaceSpec.PublicLoadBalancerName = s.clusterScope.Name() } err = s.networkInterfacesSvc.Delete(s.clusterScope.Context, networkInterfaceSpec) @@ -196,8 +199,9 @@ func (s *azureMachineService) getVirtualMachineZone() (string, error) { func (s *azureMachineService) reconcileNetworkInterface(nicName string) error { networkInterfaceSpec := &networkinterfaces.Spec{ - Name: nicName, - VnetName: s.clusterScope.Vnet().Name, + Name: nicName, + VnetName: s.clusterScope.Vnet().Name, + MachineRole: s.machineScope.Role(), } if s.machineScope.AzureMachine.Spec.AllocatePublicIP == true { @@ -209,6 +213,7 @@ func (s *azureMachineService) reconcileNetworkInterface(nicName string) error { switch role := s.machineScope.Role(); role { case infrav1.Node: networkInterfaceSpec.SubnetName = s.clusterScope.NodeSubnet().Name + networkInterfaceSpec.PublicLoadBalancerName = s.clusterScope.Name() case infrav1.ControlPlane: networkInterfaceSpec.SubnetName = s.clusterScope.ControlPlaneSubnet().Name networkInterfaceSpec.PublicLoadBalancerName = azure.GeneratePublicLBName(s.clusterScope.Name()) diff --git a/exp/controllers/azuremachinepool_controller.go b/exp/controllers/azuremachinepool_controller.go index 852d9d18738..148e3147a01 100644 --- a/exp/controllers/azuremachinepool_controller.go +++ b/exp/controllers/azuremachinepool_controller.go @@ -490,20 +490,21 @@ func (s *azureMachinePoolService) CreateOrUpdate() (*infrav1exp.VMSS, error) { } vmssSpec := &scalesets.Spec{ - Name: s.machinePoolScope.Name(), - ResourceGroup: s.clusterScope.ResourceGroup(), - Location: s.clusterScope.Location(), - ClusterName: s.clusterScope.Name(), - MachinePoolName: s.machinePoolScope.Name(), - Sku: ampSpec.Template.VMSize, - Capacity: replicas, - SSHKeyData: string(decoded), - Image: image, - OSDisk: ampSpec.Template.OSDisk, - CustomData: bootstrapData, - AdditionalTags: s.machinePoolScope.AdditionalTags(), - SubnetID: s.clusterScope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, - AcceleratedNetworking: ampSpec.Template.AcceleratedNetworking, + Name: s.machinePoolScope.Name(), + ResourceGroup: s.clusterScope.ResourceGroup(), + Location: s.clusterScope.Location(), + ClusterName: s.clusterScope.Name(), + MachinePoolName: s.machinePoolScope.Name(), + Sku: ampSpec.Template.VMSize, + Capacity: replicas, + SSHKeyData: string(decoded), + Image: image, + OSDisk: ampSpec.Template.OSDisk, + CustomData: bootstrapData, + AdditionalTags: s.machinePoolScope.AdditionalTags(), + SubnetID: s.clusterScope.AzureCluster.Spec.NetworkSpec.Subnets[0].ID, + PublicLoadBalancerName: s.clusterScope.Name(), + AcceleratedNetworking: ampSpec.Template.AcceleratedNetworking, } err = s.virtualMachinesScaleSetSvc.Reconcile(context.TODO(), vmssSpec)