diff --git a/PROJECT b/PROJECT index f07315b7..c32601ad 100644 --- a/PROJECT +++ b/PROJECT @@ -1,7 +1,10 @@ -version: "2" domain: keikoproj.io repo: _/tmp/instancemanager resources: - group: instancemgr - version: v1alpha1 kind: InstanceGroup + version: v1alpha1 +- group: instancemgr + kind: VerticalScalingPolicy + version: v1alpha1 +version: "2" diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index 70a8af32..6f2d8aa7 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -95,6 +95,11 @@ const ( UpgradeLockedAnnotationKey = "instancemgr.keikoproj.io/lock-upgrades" ) +const ( + InstanceGroupNameAnnotationKey = "instancemgr.keikoproj.io/instancegroup" + InstanceGroupNamespaceAnnotationKey = "instancemgr.keikoproj.io/instancegroup-namespace" +) + var ( Strategies = []string{CRDStrategyName, RollingUpdateStrategyName, ManagedStrategyName} Provisioners = []string{ @@ -351,6 +356,7 @@ type InstanceGroupStatus struct { CurrentMax int `json:"currentMax,omitempty"` ActiveLaunchConfigurationName string `json:"activeLaunchConfigurationName,omitempty"` ActiveLaunchTemplateName string `json:"activeLaunchTemplateName,omitempty"` + ActiveInstanceType string `json:"activeInstanceType,omitempty"` LatestTemplateVersion string `json:"latestTemplateVersion,omitempty"` ActiveScalingGroupName string `json:"activeScalingGroupName,omitempty"` NodesArn string `json:"nodesInstanceRoleArn,omitempty"` @@ -579,9 +585,6 @@ func (c *EKSConfiguration) Validate() error { if common.StringEmpty(c.Image) { return errors.Errorf("validation failed, 'image' is a required parameter") } - if common.StringEmpty(c.InstanceType) { - return errors.Errorf("validation failed, 'instanceType' is a required parameter") - } if common.StringEmpty(c.KeyPairName) { return errors.Errorf("validation failed, 'keyPair' is a required parameter") } @@ -1168,6 +1171,10 @@ func (status *InstanceGroupStatus) SetConditions(conditions []InstanceGroupCondi status.Conditions = conditions } +func (status *InstanceGroupStatus) SetActiveInstanceType(instanceType string) { + status.ActiveInstanceType = instanceType +} + func (strategy *AwsUpgradeStrategy) GetType() string { return strategy.Type } diff --git a/api/v1alpha1/verticalscalingpolicy_types.go b/api/v1alpha1/verticalscalingpolicy_types.go new file mode 100644 index 00000000..f0baa98b --- /dev/null +++ b/api/v1alpha1/verticalscalingpolicy_types.go @@ -0,0 +1,248 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VerticalScalingPolicySpec defines the desired state of VerticalScalingPolicy +type VerticalScalingPolicySpec struct { + InstanceFamily string `json:"instanceFamily,omitempty"` + Resources *corev1.ResourceRequirements `json:"resources"` + Targets []*corev1.ObjectReference `json:"scaleTargetsRef"` + Behavior *BehaviorSpec `json:"behavior"` +} + +type BehaviorSpec struct { + ScaleDown *ScalingSpec `json:"scaleDown,omitempty"` + ScaleUp *ScalingSpec `json:"scaleUp,omitempty"` +} + +type ScalingSpec struct { + StabilizationWindowSeconds int `json:"stabilizationWindowSeconds,omitempty"` + Policies []*PolicySpec `json:"policies,omitempty"` +} + +type PolicySpec struct { + Type UtilizationType `json:"type,omitempty"` + Value int `json:"value,omitempty"` + PeriodSeconds int `json:"periodSeconds,omitempty"` +} + +// VerticalScalingPolicyStatus defines the observed state of VerticalScalingPolicy +type VerticalScalingPolicyStatus struct { + CurrentState string `json:"currentState,omitempty"` + // store last reconcile time to check with stabilizationWindow whether to perform next scale up/down + TargetStatuses map[string]*TargetStatus `json:"targetStatuses,omitempty"` +} + +type TargetStatus struct { + State string `json:"state,omitempty"` + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + DesiredInstanceType string `json:"desiredInstanceType,omitempty"` + Conditions []*UtilizationCondition `json:"conditions,omitempty"` +} + +type UtilizationType string + +const ( + CPUUtilizationPercent UtilizationType = "CPUUtilizationPercentage" + MemoryUtilizationPercent UtilizationType = "MemoryUtilizationPercentage" + NodesCountUtilizationPercent UtilizationType = "NodesCountUtilizationPercentage" + + CPUBelowScaleDownThreshold UtilizationType = "CPUBelowScaleDownThreshold" + MemoryBelowScaleDownThreshold UtilizationType = "MemoryBelowScaleDownThreshold" + + CPUAboveScaleUpThreshold UtilizationType = "CPUAboveScaleUpThreshold" + MemoryAboveScaleUpThreshold UtilizationType = "MemoryAboveScaleUpThreshold" + NodesCountAboveScaleUpThreshold UtilizationType = "NodesCountAboveScaleUpThreshold" +) + +var validScaleUpPolicyTypes = []UtilizationType{CPUUtilizationPercent, MemoryUtilizationPercent, NodesCountUtilizationPercent} +var validScaleDownPolicyTypes = []UtilizationType{CPUUtilizationPercent, MemoryUtilizationPercent} + +// NodeCondition contains condition information for a node. +type UtilizationCondition struct { + // Type of utilization condition. + Type UtilizationType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=UtilizationType"` + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"` + // Last time we got an update on a given condition. + // +optional + LastHeartbeatTime metav1.Time `json:"lastHeartbeatTime,omitempty" protobuf:"bytes,3,opt,name=lastHeartbeatTime"` + // Last time the condition transit from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"` + // (brief) reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"` + // Human readable message indicating details about last transition. + // +optional + Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// VerticalScalingPolicy is the Schema for the verticalscalingpolicies API +type VerticalScalingPolicy struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec *VerticalScalingPolicySpec `json:"spec,omitempty"` + Status *VerticalScalingPolicyStatus `json:"status,omitempty"` +} + +func (v *VerticalScalingPolicy) InstanceFamily() (string, bool) { + return v.Spec.InstanceFamily, v.Spec.InstanceFamily != "" +} + +//+kubebuilder:object:root=true + +// VerticalScalingPolicyList contains a list of VerticalScalingPolicy +type VerticalScalingPolicyList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []VerticalScalingPolicy `json:"items"` +} + +func init() { + SchemeBuilder.Register(&VerticalScalingPolicy{}, &VerticalScalingPolicyList{}) +} + +func (scalingSpec *ScalingSpec) GetPolicy(name UtilizationType) *PolicySpec { + for _, policy := range scalingSpec.Policies { + if policy.Type == name { + return policy + } + } + + return nil +} + +func (status *VerticalScalingPolicyStatus) GetCondition(igName string, t UtilizationType) *UtilizationCondition { + for _, condition := range status.TargetStatuses[igName].Conditions { + if condition.Type == t { + return condition + } + } + return &UtilizationCondition{} +} + +func (status *VerticalScalingPolicyStatus) SetCondition(igName string, condition *UtilizationCondition) { + for _, c := range status.TargetStatuses[igName].Conditions { + if condition.Type == c.Type { + c = condition + break + } + } +} + +func (status *VerticalScalingPolicyStatus) SetConditions(igName string, conditions []*UtilizationCondition) { + status.TargetStatuses[igName].Conditions = conditions +} + +func (vsp *VerticalScalingPolicy) Validate() error { + s := vsp.Spec + + if err := s.Validate(); err != nil { + return err + } + + return nil +} + +func (s *VerticalScalingPolicySpec) Validate() error { + if s.Behavior == nil { + return errors.Errorf("validation failed, behavior not provided in spec") + } + + scaleUpSpec := s.getScaleUpSpec() + if scaleUpSpec != nil { + if err := scaleUpSpec.Validate("scaleup"); err != nil { + return err + } + } + + scaleDownSpec := s.getScaleDownSpec() + if scaleDownSpec != nil { + if err := scaleDownSpec.Validate("scaledown"); err != nil { + return err + } + } + + return nil +} + +func (s *ScalingSpec) Validate(direction string) error { + if s.Policies == nil { + return errors.Errorf("validation failed, no policies defined in scaling behaviors") + } + + if s.StabilizationWindowSeconds < 0 { + return errors.Errorf("validation failed, invalid stabilizationWindowSeconds in scaling behaviors , must be an integer >0") + } + + var validPolicyTypes []UtilizationType + if direction == "scaleup" { + validPolicyTypes = validScaleUpPolicyTypes + } else if direction == "scaledown" { + validPolicyTypes = validScaleDownPolicyTypes + } + for _, policy := range s.Policies { + err := policy.Validate(validPolicyTypes) + if err != nil { + return err + } + } + + return nil +} + +func (s *PolicySpec) Validate(validPolicyTypes []UtilizationType) error { + if !isValidUtilizationType(validPolicyTypes, s.Type) { + return errors.Errorf("validation failed, invalid policy type %s in scaling behaviors, valid types are: %s", s.Type, validPolicyTypes) + } + if strings.Contains(string(s.Type), "Percent") { + if s.Value < 0 || s.Value > 100 { + return errors.Errorf("validation failed, invalid policy percentage value %d for policy type %s in scaling behaviors, value must be from 0-100", s.Value, s.Type) + } + } + + return nil +} + +func isValidUtilizationType(validTypes []UtilizationType, typeName UtilizationType) bool { + for _, validType := range validTypes { + if typeName == validType { + return true + } + } + return false +} + +func (s *VerticalScalingPolicySpec) getScaleUpSpec() *ScalingSpec { + return s.Behavior.ScaleUp +} + +func (s *VerticalScalingPolicySpec) getScaleDownSpec() *ScalingSpec { + return s.Behavior.ScaleDown +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 83dce64f..0991814d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -51,6 +51,31 @@ func (in *AwsUpgradeStrategy) DeepCopy() *AwsUpgradeStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BehaviorSpec) DeepCopyInto(out *BehaviorSpec) { + *out = *in + if in.ScaleDown != nil { + in, out := &in.ScaleDown, &out.ScaleDown + *out = new(ScalingSpec) + (*in).DeepCopyInto(*out) + } + if in.ScaleUp != nil { + in, out := &in.ScaleUp, &out.ScaleUp + *out = new(ScalingSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BehaviorSpec. +func (in *BehaviorSpec) DeepCopy() *BehaviorSpec { + if in == nil { + return nil + } + out := new(BehaviorSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapOptions) DeepCopyInto(out *BootstrapOptions) { *out = *in @@ -633,6 +658,21 @@ func (in *PlacementSpec) DeepCopy() *PlacementSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PolicySpec) DeepCopyInto(out *PolicySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PolicySpec. +func (in *PolicySpec) DeepCopy() *PolicySpec { + if in == nil { + return nil + } + out := new(PolicySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { *out = *in @@ -653,6 +693,59 @@ func (in *RollingUpdateStrategy) DeepCopy() *RollingUpdateStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScalingSpec) DeepCopyInto(out *ScalingSpec) { + *out = *in + if in.Policies != nil { + in, out := &in.Policies, &out.Policies + *out = make([]*PolicySpec, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(PolicySpec) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScalingSpec. +func (in *ScalingSpec) DeepCopy() *ScalingSpec { + if in == nil { + return nil + } + out := new(ScalingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TargetStatus) DeepCopyInto(out *TargetStatus) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]*UtilizationCondition, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(UtilizationCondition) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetStatus. +func (in *TargetStatus) DeepCopy() *TargetStatus { + if in == nil { + return nil + } + out := new(TargetStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserDataStage) DeepCopyInto(out *UserDataStage) { *out = *in @@ -668,6 +761,156 @@ func (in *UserDataStage) DeepCopy() *UserDataStage { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UtilizationCondition) DeepCopyInto(out *UtilizationCondition) { + *out = *in + in.LastHeartbeatTime.DeepCopyInto(&out.LastHeartbeatTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UtilizationCondition. +func (in *UtilizationCondition) DeepCopy() *UtilizationCondition { + if in == nil { + return nil + } + out := new(UtilizationCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VerticalScalingPolicy) DeepCopyInto(out *VerticalScalingPolicy) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + if in.Spec != nil { + in, out := &in.Spec, &out.Spec + *out = new(VerticalScalingPolicySpec) + (*in).DeepCopyInto(*out) + } + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = new(VerticalScalingPolicyStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VerticalScalingPolicy. +func (in *VerticalScalingPolicy) DeepCopy() *VerticalScalingPolicy { + if in == nil { + return nil + } + out := new(VerticalScalingPolicy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VerticalScalingPolicy) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VerticalScalingPolicyList) DeepCopyInto(out *VerticalScalingPolicyList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VerticalScalingPolicy, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VerticalScalingPolicyList. +func (in *VerticalScalingPolicyList) DeepCopy() *VerticalScalingPolicyList { + if in == nil { + return nil + } + out := new(VerticalScalingPolicyList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VerticalScalingPolicyList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VerticalScalingPolicySpec) DeepCopyInto(out *VerticalScalingPolicySpec) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.Targets != nil { + in, out := &in.Targets, &out.Targets + *out = make([]*v1.ObjectReference, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(v1.ObjectReference) + **out = **in + } + } + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(BehaviorSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VerticalScalingPolicySpec. +func (in *VerticalScalingPolicySpec) DeepCopy() *VerticalScalingPolicySpec { + if in == nil { + return nil + } + out := new(VerticalScalingPolicySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VerticalScalingPolicyStatus) DeepCopyInto(out *VerticalScalingPolicyStatus) { + *out = *in + if in.TargetStatuses != nil { + in, out := &in.TargetStatuses, &out.TargetStatuses + *out = make(map[string]*TargetStatus, len(*in)) + for key, val := range *in { + var outVal *TargetStatus + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(TargetStatus) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VerticalScalingPolicyStatus. +func (in *VerticalScalingPolicyStatus) DeepCopy() *VerticalScalingPolicyStatus { + if in == nil { + return nil + } + out := new(VerticalScalingPolicyStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WarmPoolSpec) DeepCopyInto(out *WarmPoolSpec) { *out = *in diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml index 56a02cea..71152a51 100644 --- a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml +++ b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml @@ -57,10 +57,14 @@ spec: description: InstanceGroup is the Schema for the instancegroups API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -203,20 +207,27 @@ spec: type: array taints: items: - description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + description: The node this Taint is attached to has the + "effect" on any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + description: Required. The effect of the taint on pods + that do not tolerate the taint. Valid effects are + NoSchedule, PreferNoSchedule and NoExecute. type: string key: - description: Required. The taint key to be applied to a node. + description: Required. The taint key to be applied to + a node. type: string timeAdded: - description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + description: TimeAdded represents the time at which + the taint was added. It is only written for NoExecute + taints. format: date-time type: string value: - description: The taint value corresponding to the taint key. + description: The taint value corresponding to the taint + key. type: string required: - effect @@ -382,7 +393,8 @@ spec: provisioner: type: string strategy: - description: AwsUpgradeStrategy defines the upgrade strategy of an AWS Instance Group + description: AwsUpgradeStrategy defines the upgrade strategy of an + AWS Instance Group properties: crd: properties: @@ -416,6 +428,8 @@ spec: status: description: InstanceGroupStatus defines the schema of resource Status properties: + activeInstanceType: + type: string activeLaunchConfigurationName: type: string activeLaunchTemplateName: @@ -424,7 +438,8 @@ spec: type: string conditions: items: - description: InstanceGroupConditions describes the conditions of the InstanceGroup + description: InstanceGroupConditions describes the conditions of + the InstanceGroup properties: status: type: string diff --git a/config/crd/bases/instancemgr.keikoproj.io_verticalscalingpolicies.yaml b/config/crd/bases/instancemgr.keikoproj.io_verticalscalingpolicies.yaml new file mode 100644 index 00000000..c43db091 --- /dev/null +++ b/config/crd/bases/instancemgr.keikoproj.io_verticalscalingpolicies.yaml @@ -0,0 +1,234 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: verticalscalingpolicies.instancemgr.keikoproj.io +spec: + group: instancemgr.keikoproj.io + names: + kind: VerticalScalingPolicy + listKind: VerticalScalingPolicyList + plural: verticalscalingpolicies + singular: verticalscalingpolicy + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: VerticalScalingPolicy is the Schema for the verticalscalingpolicies + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VerticalScalingPolicySpec defines the desired state of VerticalScalingPolicy + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + type: integer + type: + type: string + value: + type: integer + type: object + type: array + stabilizationWindowSeconds: + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + type: integer + type: + type: string + value: + type: integer + type: object + type: array + stabilizationWindowSeconds: + type: integer + type: object + type: object + instanceFamily: + type: string + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + scaleTargetsRef: + items: + description: 'ObjectReference contains enough information to let + you inspect or modify the referred object. --- New uses of this + type are discouraged because of difficulty describing its usage + when embedded in APIs. 1. Ignored fields. It includes many fields + which are not generally honored. For instance, ResourceVersion + and FieldPath are both very rarely valid in actual usage. 2. + Invalid usage help. It is impossible to add specific help for + individual usage. In most embedded usages, there are particular restrictions + like, "must refer only to types A and B" or "UID not honored" + or "name must be restricted". Those cannot be well described + when embedded. 3. Inconsistent validation. Because the usages + are different, the validation rules are different by usage, which + makes it hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency is + on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this type will + affect numerous schemas. Don''t make new APIs embed an underspecified + API type they do not control. Instead of using this type, create + a locally provided and used type that is well-focused on your + reference. For example, ServiceReferences for admission registration: + https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + required: + - behavior + - resources + - scaleTargetsRef + type: object + status: + description: VerticalScalingPolicyStatus defines the observed state of + VerticalScalingPolicy + properties: + currentState: + type: string + targetStatuses: + additionalProperties: + properties: + conditions: + items: + description: NodeCondition contains condition information + for a node. + properties: + lastHeartbeatTime: + description: Last time we got an update on a given condition. + format: date-time + type: string + lastTransitionTime: + description: Last time the condition transit from one + status to another. + format: date-time + type: string + message: + description: Human readable message indicating details + about last transition. + type: string + reason: + description: (brief) reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of utilization condition. + type: string + required: + - status + - type + type: object + type: array + desiredInstanceType: + type: string + lastTransitionTime: + format: date-time + type: string + state: + type: string + type: object + description: store last reconcile time to check with stabilizationWindow + whether to perform next scale up/down + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index d9ccacfa..6b7b0919 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/instancemgr.keikoproj.io_instancegroups.yaml +- bases/instancemgr.keikoproj.io_verticalscalingpolicies.yaml # +kubebuilder:scaffold:crdkustomizeresource #patches: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 69ecb8ce..3408cc4f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -74,3 +74,23 @@ rules: - get - patch - update +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies/status + verbs: + - get + - patch + - update diff --git a/config/rbac/verticalscalingpolicy_editor_role.yaml b/config/rbac/verticalscalingpolicy_editor_role.yaml new file mode 100644 index 00000000..2d61eb95 --- /dev/null +++ b/config/rbac/verticalscalingpolicy_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit verticalscalingpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: verticalscalingpolicy-editor-role +rules: +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies/status + verbs: + - get diff --git a/config/rbac/verticalscalingpolicy_viewer_role.yaml b/config/rbac/verticalscalingpolicy_viewer_role.yaml new file mode 100644 index 00000000..f9f8b575 --- /dev/null +++ b/config/rbac/verticalscalingpolicy_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view verticalscalingpolicies. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: verticalscalingpolicy-viewer-role +rules: +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies + verbs: + - get + - list + - watch +- apiGroups: + - instancemgr.keikoproj.io + resources: + - verticalscalingpolicies/status + verbs: + - get diff --git a/config/samples/instancemgr_v1alpha1_verticalscalingpolicy.yaml b/config/samples/instancemgr_v1alpha1_verticalscalingpolicy.yaml new file mode 100644 index 00000000..5ed6bb60 --- /dev/null +++ b/config/samples/instancemgr_v1alpha1_verticalscalingpolicy.yaml @@ -0,0 +1,31 @@ +apiVersion: instancemgr.keikoproj.io/v1alpha1 +kind: VerticalScalingPolicy +metadata: + name: verticalscalingpolicy-sample +spec: + instanceFamily: m5 + resources: + requests: + mem: 8Gi + cpu: 2 + limits: + mem: 64Gi + cpu: 16 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: CPUUtilizationPercentage + value: 20 + periodSeconds: 1800 + scaleUp: + stabilizationWindowSeconds: 300 + policies: + - type: NodeUtilizationPercentage + value: 90 + periodSeconds: 60 + scaleTargetRef: + apiVersion: instancemgr.keikoproj.io/v1alpha1 + kind: InstanceGroup + name: my-instance-group + diff --git a/controllers/common/utils.go b/controllers/common/utils.go index 892de8c7..8f038297 100644 --- a/controllers/common/utils.go +++ b/controllers/common/utils.go @@ -67,6 +67,18 @@ func ContainsString(slice []string, s string) bool { return false } +func RemoveDuplicateValues(strSlice []string) []string { + keys := make(map[string]bool) + list := []string{} + for _, entry := range strSlice { + if _, value := keys[entry]; !value { + keys[entry] = true + list = append(list, entry) + } + } + return list +} + func ContainsEqualFoldSubstring(str, substr string) bool { x := strings.ToLower(str) y := strings.ToLower(substr) @@ -326,3 +338,13 @@ func IntOrStrValue(x *intstr.IntOrString) int { func Int64ToStr(x int64) string { return strconv.FormatInt(x, 10) } + +// GetStringIndexInSlice returns index of string 's' if a given slice 'slice' contains string 's', otherwise return -1 +func GetStringIndexInSlice(slice []string, s string) int { + for index, item := range slice { + if item == s { + return index + } + } + return -1 +} diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index 881a2f38..b719d03f 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -45,7 +45,7 @@ type InstanceGroupReconciler struct { client.Client SpotRecommendationTime float64 ConfigNamespace string - NodeRelabel bool + WithNodeWatch bool Log logr.Logger MaxParallel int Auth *InstanceGroupAuthenticator @@ -55,6 +55,7 @@ type InstanceGroupReconciler struct { ConfigRetention int Metrics *common.MetricsCollector DisableWinClusterInjection bool + ManagerContext *SharedContext } type InstanceGroupAuthenticator struct { @@ -175,6 +176,7 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque status.SetConfigHash("") } } + r.ManagerContext.InstanceGroups[req.NamespacedName.String()] = input.InstanceGroup provisionerKind := strings.ToLower(input.InstanceGroup.Spec.Provisioner) @@ -201,6 +203,11 @@ func (r *InstanceGroupReconciler) Reconcile(ctxt context.Context, req ctrl.Reque return ctrl.Result{}, errors.Wrapf(err, "provisioner %v reconcile failed", provisionerKind) } + computedType := r.ManagerContext.GetComputedType(req.NamespacedName.String()) + if !common.StringEmpty(computedType) { + input.InstanceGroup.Spec.EKSSpec.EKSConfiguration.InstanceType = computedType + } + if err = HandleReconcileRequest(ctx); err != nil { ctx.SetState(v1alpha1.ReconcileErr) r.PatchStatus(input.InstanceGroup, statusPatch) diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index 7e20cbf6..0bfe7521 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -54,10 +54,10 @@ var ( InstanceMgrLifecycleLabel = "instancemgr.keikoproj.io/lifecycle" InstanceMgrImageLabel = "instancemgr.keikoproj.io/image" - AllowedOsFamilies = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2} - DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"} - CNIManagedPolicy = "AmazonEKS_CNI_Policy" - SupportedArchitectures = []string{"x86_64", "arm64"} + AllowedOsFamilies = []string{OsFamilyWindows, OsFamilyBottleRocket, OsFamilyAmazonLinux2} + DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"} + CNIManagedPolicy = "AmazonEKS_CNI_Policy" + SupportedArchitectures = []string{"x86_64", "arm64"} ) // New constructs a new instance group provisioner of EKS type diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go index da3076ff..4077efc7 100644 --- a/controllers/provisioners/eks/helpers.go +++ b/controllers/provisioners/eks/helpers.go @@ -479,6 +479,8 @@ func (ctx *EksInstanceGroupContext) GetComputedLabels() map[string]string { } labelMap[InstanceMgrImageLabel] = configuration.GetImage() + labelMap[v1alpha1.InstanceGroupNameAnnotationKey] = fmt.Sprintf("%v", instanceGroup.GetName()) + labelMap[v1alpha1.InstanceGroupNamespaceAnnotationKey] = fmt.Sprintf("%v", instanceGroup.GetNamespace()) return labelMap } @@ -696,6 +698,30 @@ func (ctx *EksInstanceGroupContext) findTargetScalingGroup(groups []*autoscaling return nil } +func (ctx *EksInstanceGroupContext) UpdateActiveInstanceType() { + var ( + state = ctx.GetDiscoveredState() + instanceGroup = ctx.GetInstanceGroup() + status = instanceGroup.GetStatus() + scalingGroup = state.GetScalingGroup() + ) + + if scalingGroup == nil { + return + } + + scalingTypes := make([]string, 0) + for _, instance := range scalingGroup.Instances { + scalingTypes = append(scalingTypes, aws.StringValue(instance.InstanceType)) + } + + scalingTypes = common.RemoveDuplicateValues(scalingTypes) + + if len(scalingTypes) == 1 { + status.SetActiveInstanceType(scalingTypes[0]) + } +} + func (ctx *EksInstanceGroupContext) UpdateNodeReadyCondition() bool { var ( state = ctx.GetDiscoveredState() diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go index 1e4b394a..90af0b25 100644 --- a/controllers/provisioners/eks/helpers_test.go +++ b/controllers/provisioners/eks/helpers_test.go @@ -189,7 +189,7 @@ func TestGetBasicUserDataWindows(t *testing.T) { ctx := MockContext(ig, k, w) configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{ - MaxPods: 4, + MaxPods: 4, ContainerRuntime: "containerd", } configuration.Labels = map[string]string{ diff --git a/controllers/provisioners/eks/update.go b/controllers/provisioners/eks/update.go index 34b854fc..8f76683f 100644 --- a/controllers/provisioners/eks/update.go +++ b/controllers/provisioners/eks/update.go @@ -138,9 +138,13 @@ func (ctx *EksInstanceGroupContext) Update() error { if nodesReady { ctx.SetState(v1alpha1.ReconcileModified) } + + fmt.Println("test") if rotationNeeded { ctx.SetState(v1alpha1.ReconcileInitUpgrade) } else { + fmt.Println("test2") + ctx.UpdateActiveInstanceType() status.SetStrategyRetryCount(0) } diff --git a/controllers/reconcilers.go b/controllers/reconcilers.go index e149d854..d9aad679 100644 --- a/controllers/reconcilers.go +++ b/controllers/reconcilers.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "strings" + "sync" "time" v1alpha1 "github.com/keikoproj/instance-manager/api/v1alpha1" @@ -36,6 +37,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/source" ) @@ -45,7 +47,7 @@ const ( ) func (r *InstanceGroupReconciler) SetupWithManager(mgr ctrl.Manager) error { - switch r.NodeRelabel { + switch r.WithNodeWatch { case true: return ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.InstanceGroup{}). @@ -53,6 +55,7 @@ func (r *InstanceGroupReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(r.nodeReconciler)). Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.configMapReconciler)). Watches(&source.Kind{Type: &corev1.Namespace{}}, handler.EnqueueRequestsFromMapFunc(r.namespaceReconciler)). + Watches(&source.Channel{Source: r.ManagerContext.InstanceGroupEvents}, handler.EnqueueRequestsFromMapFunc(genericReconciler)). WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxParallel}). Complete(r) default: @@ -61,11 +64,53 @@ func (r *InstanceGroupReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches(&source.Kind{Type: &corev1.Event{}}, handler.EnqueueRequestsFromMapFunc(r.spotEventReconciler)). Watches(&source.Kind{Type: &corev1.ConfigMap{}}, handler.EnqueueRequestsFromMapFunc(r.configMapReconciler)). Watches(&source.Kind{Type: &corev1.Namespace{}}, handler.EnqueueRequestsFromMapFunc(r.namespaceReconciler)). + Watches(&source.Channel{Source: r.ManagerContext.InstanceGroupEvents}, handler.EnqueueRequestsFromMapFunc(genericReconciler)). WithOptions(controller.Options{MaxConcurrentReconciles: r.MaxParallel}). Complete(r) } } +// SetupWithManager sets up the controller with the Manager. +func (r *VerticalScalingPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.VerticalScalingPolicy{}). + Watches(&source.Channel{Source: r.Resync}, handler.EnqueueRequestsFromMapFunc(r.resyncAll)). + Complete(r) +} + +func genericReconciler(obj client.Object) []ctrl.Request { + return []ctrl.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, + }, + } +} + +func (r *VerticalScalingPolicyReconciler) resyncAll(obj client.Object) []ctrl.Request { + var ( + vspList = &v1alpha1.VerticalScalingPolicyList{} + requests []ctrl.Request + ) + err := r.List(context.Background(), vspList) + if err != nil { + ctrl.Log.Error(err, "failed to list vertical scaling policies") + return nil + } + + for _, v := range vspList.Items { + requests = append(requests, ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: v.GetNamespace(), + Name: v.GetName(), + }, + }) + } + return requests +} + func (r *InstanceGroupReconciler) configMapReconciler(obj client.Object) []ctrl.Request { var ( name = obj.GetName() @@ -201,6 +246,19 @@ func (r *InstanceGroupReconciler) nodeReconciler(obj client.Object) []ctrl.Reque roleLabelKey = "kubernetes.io/role" bootstrapLabelKey = "node.kubernetes.io/role" ) + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + r.Log.Error(err, "could not convert runtime object to unstructured") + return nil + } + + node := &corev1.Node{} + if err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredObj, node); err != nil { + r.Log.Error(err, "could not convert unstructured object to node") + return nil + } + + r.ManagerContext.Nodes[nodeName] = node // if node already has a role label, don't modify it if _, ok := nodeLabels[roleLabelKey]; ok { @@ -284,3 +342,63 @@ func (r *InstanceGroupReconciler) spotEventReconciler(obj client.Object) []ctrl. }, } } + +type SharedContext struct { + sync.RWMutex + + InstanceGroupEvents chan event.GenericEvent + + // "instance-manager/my-ig-1": []Node{} + Nodes map[string]*corev1.Node + + InstanceGroups map[string]*v1alpha1.InstanceGroup + + // "instance-manager/my-vsp": VSP{} + Policies map[string]v1alpha1.VerticalScalingPolicy + + // "instance-manager/my-ig-1": "m5.xlarge" + ComputedTypes map[string]string +} + +func (m *SharedContext) UpsertPolicy(policy *v1alpha1.VerticalScalingPolicy) { + m.Lock() + defer m.Unlock() + namespacedName := fmt.Sprintf("%v/%v", policy.GetNamespace(), policy.GetName()) + m.Policies[namespacedName] = *policy +} + +func (m *SharedContext) RemovePolicy(namespacedName string) { + m.Lock() + defer m.Unlock() + delete(m.Policies, namespacedName) +} + +func (m *SharedContext) GetPolicy(namespacedName string) v1alpha1.VerticalScalingPolicy { + m.RLock() + defer m.RUnlock() + if v, ok := m.Policies[namespacedName]; ok { + return v + } + return v1alpha1.VerticalScalingPolicy{} +} + +func (m *SharedContext) UpsertComputedType(targetName, t string) { + m.Lock() + defer m.Unlock() + m.ComputedTypes[targetName] = t +} + +func (m *SharedContext) RemoveComputedType(targetName string) { + m.Lock() + defer m.Unlock() + delete(m.ComputedTypes, targetName) +} + +func (m *SharedContext) GetComputedType(namespacedName string) string { + m.RLock() + defer m.RUnlock() + if v, ok := m.ComputedTypes[namespacedName]; ok { + return v + } + return "" +} diff --git a/controllers/verticalscalingpolicy_controller.go b/controllers/verticalscalingpolicy_controller.go new file mode 100644 index 00000000..d6b9922f --- /dev/null +++ b/controllers/verticalscalingpolicy_controller.go @@ -0,0 +1,468 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/go-logr/logr" + "github.com/keikoproj/instance-manager/api/v1alpha1" + "github.com/keikoproj/instance-manager/controllers/common" + "github.com/keikoproj/instance-manager/controllers/providers/kubernetes" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + resource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +// VerticalScalingPolicyReconciler reconciles a VerticalScalingPolicy object +type VerticalScalingPolicyReconciler struct { + client.Client + Log logr.Logger + Auth *InstanceGroupAuthenticator + ManagerContext *SharedContext + Resync chan event.GenericEvent +} + +//+kubebuilder:rbac:groups=instancemgr.keikoproj.io,resources=verticalscalingpolicies,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=instancemgr.keikoproj.io,resources=verticalscalingpolicies/status,verbs=get;update;patch + +func (r *VerticalScalingPolicyReconciler) Reconcile(ctxt context.Context, req ctrl.Request) (ctrl.Result, error) { + _ = context.Background() + _ = r.Log.WithValues("verticalscalingpolicy", req.NamespacedName) + driftedTargets := make(map[string]bool) + + vsp := &v1alpha1.VerticalScalingPolicy{} + err := r.Get(ctxt, req.NamespacedName, vsp) + if err != nil { + if kerrors.IsNotFound(err) { + r.Log.Info("verticalscalingpolicy not found", "verticalscalingpolicy", req.NamespacedName) + r.ManagerContext.RemovePolicy(req.NamespacedName.String()) + return ctrl.Result{}, nil + } + r.Log.Error(err, "reconcile failed", "verticalscalingpolicy", req.NamespacedName) + return ctrl.Result{}, err + } + + // update the policies map + r.ManagerContext.UpsertPolicy(vsp) + + // Which type ranges should we use? + + // Get instance types info + // Call AWS (via caching) to get instance types info + types, err := r.Auth.Aws.DescribeInstanceTypes() + if err != nil { + return ctrl.Result{}, err + } + + instanceTypeRange, err := r.calculateInstanceTypeRange(vsp, types) + if err != nil { + return ctrl.Result{}, err + } + instanceTypeRange.InstanceTypes = []string{ + "m5.large", + "m5.xlarge", + "m5.2xlarge", + "m5.4xlarge", + } + + // decide on computed type + + // come up with matching instance type according to resources/requests/limits and instance family + + // Should we scale? + for _, ig := range vsp.Spec.Targets { + namespacedName := fmt.Sprintf("%v/%v", ig.Namespace, ig.Name) + igObj := r.ManagerContext.InstanceGroups[namespacedName] + if igObj == nil { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + namespacedIGName := fmt.Sprintf("%v/%v", igObj.GetNamespace(), igObj.GetName()) + var nodesOfIG = make([]*corev1.Node, 0) + + for _, node := range r.ManagerContext.Nodes { + nodeLabels := node.GetLabels() + if kubernetes.HasAnnotationWithValue(nodeLabels, v1alpha1.InstanceGroupNameAnnotationKey, igObj.GetName()) && kubernetes.HasAnnotationWithValue(nodeLabels, v1alpha1.InstanceGroupNamespaceAnnotationKey, igObj.GetNamespace()) { + nodesOfIG = append(nodesOfIG, node) + } + } + + currInstanceTypeIndex := common.GetStringIndexInSlice(instanceTypeRange.InstanceTypes, igObj.Status.ActiveInstanceType) + fmt.Println(instanceTypeRange.InstanceTypes) + fmt.Println(igObj.Status.ActiveInstanceType) + if currInstanceTypeIndex == -1 { + r.Log.Error(err, "reconcile failed, current instance type not found in computed types", "verticalscalingpolicy", req.NamespacedName) + return ctrl.Result{}, err + } + + if vsp.Status == nil { + vsp.Status = &v1alpha1.VerticalScalingPolicyStatus{ + TargetStatuses: map[string]*v1alpha1.TargetStatus{ + namespacedName: &v1alpha1.TargetStatus{ + Conditions: make([]*v1alpha1.UtilizationCondition, 0), + }, + }, + } + } + + // TODO: For period seconds we need to store node stats with timestamp or read and understand events + hasLargerInstanceType := len(instanceTypeRange.InstanceTypes) > currInstanceTypeIndex+2 + hasSmallerInstanceType := currInstanceTypeIndex-1 >= 0 + + totalCPUAllocatable := *resource.NewQuantity(0, resource.BinarySI) + totalCPUCapacity := *resource.NewQuantity(0, resource.BinarySI) + + totalMemoryAllocatable := *resource.NewQuantity(0, resource.BinarySI) + totalMemoryCapacity := *resource.NewQuantity(0, resource.BinarySI) + + nodesCountAboveScaleUpThreshold := vsp.Status.GetCondition(namespacedIGName, v1alpha1.NodesCountAboveScaleUpThreshold) + cpuAboveScaleUpThreshold := vsp.Status.GetCondition(namespacedIGName, v1alpha1.CPUAboveScaleUpThreshold) + memoryAboveScaleUpThreshold := vsp.Status.GetCondition(namespacedIGName, v1alpha1.MemoryAboveScaleUpThreshold) + + cpuBelowScaleDownThreshold := vsp.Status.GetCondition(namespacedIGName, v1alpha1.CPUBelowScaleDownThreshold) + memoryBelowScaleDownThreshold := vsp.Status.GetCondition(namespacedIGName, v1alpha1.MemoryBelowScaleDownThreshold) + + // Calculate current CPU utilization + // Calculate current Memory utilization + for _, node := range nodesOfIG { + totalCPUAllocatable.Add(*node.Status.Allocatable.Cpu()) + totalMemoryAllocatable.Add(*node.Status.Allocatable.Memory()) + totalCPUCapacity.Add(*node.Status.Capacity.Cpu()) + totalMemoryCapacity.Add(*node.Status.Capacity.Memory()) + } + + totalCPUCapacityFloat := totalCPUCapacity.AsApproximateFloat64() + totalCPUAllocatableFloat := totalCPUAllocatable.AsApproximateFloat64() + totalMemoryCapacityFloat := totalMemoryCapacity.AsApproximateFloat64() + totalMemoryAllocatableFloat := totalMemoryAllocatable.AsApproximateFloat64() + + currentCapacityUtilization := 100 * (totalCPUCapacityFloat - totalCPUAllocatableFloat) / totalCPUCapacityFloat + currentMemoryUtilization := 100 * (totalMemoryCapacityFloat - totalMemoryAllocatableFloat) / totalMemoryCapacityFloat + + var scaleUpOnNodesCountPolicy, scaleUpOnCpuUtilizationPolicy, scaleUpOnMemoryUtilizationPolicy *v1alpha1.PolicySpec + var scaleDownOnCpuUtilizationPolicy, scaleDownOnMemoryUtilizationPolicy *v1alpha1.PolicySpec + var scaleUp_stabilizationWindow, scaleDown_stabilizationWindow time.Duration + + if vsp.Spec.Behavior != nil { + if vsp.Spec.Behavior.ScaleUp != nil { + scaleUpOnNodesCountPolicy = vsp.Spec.Behavior.ScaleUp.GetPolicy(v1alpha1.NodesCountUtilizationPercent) + scaleUpOnCpuUtilizationPolicy = vsp.Spec.Behavior.ScaleUp.GetPolicy(v1alpha1.CPUUtilizationPercent) + scaleUpOnMemoryUtilizationPolicy = vsp.Spec.Behavior.ScaleUp.GetPolicy(v1alpha1.MemoryUtilizationPercent) + scaleUp_stabilizationWindow = time.Duration(vsp.Spec.Behavior.ScaleUp.StabilizationWindowSeconds) * time.Second + + } + if vsp.Spec.Behavior.ScaleDown != nil { + scaleDownOnCpuUtilizationPolicy = vsp.Spec.Behavior.ScaleDown.GetPolicy(v1alpha1.CPUUtilizationPercent) + scaleDownOnMemoryUtilizationPolicy = vsp.Spec.Behavior.ScaleDown.GetPolicy(v1alpha1.MemoryUtilizationPercent) + scaleDown_stabilizationWindow = time.Duration(vsp.Spec.Behavior.ScaleDown.StabilizationWindowSeconds) * time.Second + + } + } + + // If there is a larger instance type available, check if we want to vertically scale up the IG + if hasLargerInstanceType && vsp.Status != nil && time.Since(vsp.Status.TargetStatuses[namespacedIGName].LastTransitionTime.Time) > scaleUp_stabilizationWindow { + nextBiggerInstance := instanceTypeRange.InstanceTypes[currInstanceTypeIndex+1] + + if scaleUpOnNodesCountPolicy != nil { + nodesCountAboveScaleUpThreshold.LastHeartbeatTime = metav1.Now() + + // Scale up if current nodes count crosses requested threshold (close to maxSize of IG) + if len(nodesOfIG) > int(igObj.Spec.EKSSpec.MaxSize)*scaleUpOnNodesCountPolicy.Value/100 { + if nodesCountAboveScaleUpThreshold.Status == corev1.ConditionFalse { + nodesCountAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + nodesCountAboveScaleUpThreshold.Status = corev1.ConditionTrue + } + + if time.Since(nodesCountAboveScaleUpThreshold.LastTransitionTime.Time) > time.Duration(scaleUpOnNodesCountPolicy.PeriodSeconds) { + r.ManagerContext.ComputedTypes[namespacedIGName] = nextBiggerInstance + driftedTargets[namespacedIGName] = true + continue + } + + } else if nodesCountAboveScaleUpThreshold.Status == corev1.ConditionTrue { + nodesCountAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + nodesCountAboveScaleUpThreshold.Status = corev1.ConditionFalse + } + } + + if scaleUpOnCpuUtilizationPolicy != nil { + cpuAboveScaleUpThreshold.LastHeartbeatTime = metav1.Now() + + // Scale up if total CPU utilization crosses requested threshold (close to node sizing) + if 100*(totalCPUCapacityFloat-totalCPUAllocatableFloat)/totalCPUCapacityFloat > float64(scaleUpOnCpuUtilizationPolicy.Value) { + if cpuAboveScaleUpThreshold.Status == corev1.ConditionFalse { + cpuAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + cpuAboveScaleUpThreshold.Status = corev1.ConditionTrue + } + + if time.Since(cpuAboveScaleUpThreshold.LastTransitionTime.Time) > time.Duration(scaleUpOnCpuUtilizationPolicy.PeriodSeconds) { + r.ManagerContext.ComputedTypes[namespacedIGName] = nextBiggerInstance + driftedTargets[namespacedIGName] = true + continue + } + } else if cpuAboveScaleUpThreshold.Status == corev1.ConditionTrue { + cpuAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + cpuAboveScaleUpThreshold.Status = corev1.ConditionFalse + } + } + + if scaleUpOnMemoryUtilizationPolicy != nil { + memoryAboveScaleUpThreshold.LastHeartbeatTime = metav1.Now() + + if 100*(totalMemoryCapacityFloat-totalMemoryAllocatableFloat)/totalMemoryCapacityFloat > float64(scaleUpOnMemoryUtilizationPolicy.Value) { + if memoryAboveScaleUpThreshold.Status == corev1.ConditionFalse { + memoryAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + memoryAboveScaleUpThreshold.Status = corev1.ConditionTrue + } + + if time.Since(memoryAboveScaleUpThreshold.LastTransitionTime.Time) > time.Duration(scaleUpOnMemoryUtilizationPolicy.PeriodSeconds) { + r.ManagerContext.ComputedTypes[namespacedIGName] = nextBiggerInstance + driftedTargets[namespacedIGName] = true + continue + } + } else if memoryAboveScaleUpThreshold.Status == corev1.ConditionTrue { + memoryAboveScaleUpThreshold.LastTransitionTime = metav1.Now() + memoryAboveScaleUpThreshold.Status = corev1.ConditionFalse + } + } + } + + // If there is a smaller instance type available, check if we want to vertically scale down the IG + if hasSmallerInstanceType && vsp.Status != nil && time.Since(vsp.Status.TargetStatuses[namespacedIGName].LastTransitionTime.Time) > scaleDown_stabilizationWindow { + nextSmallerInstance := instanceTypeRange.InstanceTypes[currInstanceTypeIndex-1] + /** + * When we scale down, utilizations on smaller instance double + * If smaller instance utilization > scaleUpOnCpuUtilization || scaleUpOnMemoryUtilization + * => scale down is going to trigger a scale up after stabilizationWindowSeconds + * => we shouldn't scale down + * + * TODO: + * Number of smaller Instances >= Minimum number of nodes required due to affinity/antiaffinity rules + * If number of smaller instances > nodesCountUtilizationThreshold + * => scale down is going to trigger a scale up after stabilizationWindowSeconds + * => we shouldn't scale down + */ + + if scaleDownOnCpuUtilizationPolicy != nil { + cpuBelowScaleDownThreshold.LastHeartbeatTime = metav1.Now() + + if currentCapacityUtilization < float64(scaleDownOnCpuUtilizationPolicy.Value) { // Check if scale down is a possibility + if cpuBelowScaleDownThreshold.Status == corev1.ConditionFalse { + cpuBelowScaleDownThreshold.LastTransitionTime = metav1.Now() + cpuBelowScaleDownThreshold.Status = corev1.ConditionTrue + } + + if time.Since(cpuBelowScaleDownThreshold.LastTransitionTime.Time) > time.Duration(scaleDownOnCpuUtilizationPolicy.PeriodSeconds) { + // Check if eventual scale up is a possibility + if scaleUpOnCpuUtilizationPolicy != nil && currentCapacityUtilization/2 < float64(scaleUpOnCpuUtilizationPolicy.Value) { + r.ManagerContext.ComputedTypes[namespacedIGName] = nextSmallerInstance + driftedTargets[namespacedIGName] = true + continue + } + } + } else if cpuBelowScaleDownThreshold.Status == corev1.ConditionTrue { + cpuBelowScaleDownThreshold.LastTransitionTime = metav1.Now() + cpuBelowScaleDownThreshold.Status = corev1.ConditionFalse + } + } + + if scaleDownOnMemoryUtilizationPolicy != nil { + memoryBelowScaleDownThreshold.LastHeartbeatTime = metav1.Now() + + if currentMemoryUtilization < float64(scaleDownOnMemoryUtilizationPolicy.Value) { // Check if scale down is a possibility + if memoryBelowScaleDownThreshold.Status == corev1.ConditionFalse { + memoryBelowScaleDownThreshold.LastTransitionTime = metav1.Now() + memoryBelowScaleDownThreshold.Status = corev1.ConditionTrue + } + + if scaleUpOnMemoryUtilizationPolicy != nil && currentMemoryUtilization/2 < float64(scaleUpOnMemoryUtilizationPolicy.Value) { // Check if eventual scale up is a possibility + if time.Since(memoryBelowScaleDownThreshold.LastTransitionTime.Time) > time.Duration(scaleDownOnMemoryUtilizationPolicy.PeriodSeconds) { + r.ManagerContext.ComputedTypes[namespacedIGName] = nextSmallerInstance + driftedTargets[namespacedIGName] = true + continue + } + } + } else if memoryBelowScaleDownThreshold.Status == corev1.ConditionTrue { + memoryBelowScaleDownThreshold.LastTransitionTime = metav1.Now() + memoryBelowScaleDownThreshold.Status = corev1.ConditionFalse + } + } + } + + // TODO: Create a function to update status + + fmt.Println(driftedTargets) + + vsp.Status.TargetStatuses[namespacedIGName] = &v1alpha1.TargetStatus{ + LastTransitionTime: metav1.Time{Time: time.Now()}, + DesiredInstanceType: r.ManagerContext.ComputedTypes[namespacedIGName], + Conditions: []*v1alpha1.UtilizationCondition{ + nodesCountAboveScaleUpThreshold, + cpuAboveScaleUpThreshold, + memoryAboveScaleUpThreshold, + cpuBelowScaleDownThreshold, + memoryBelowScaleDownThreshold, + }, + } + + // State: ig reconcilation state TODO: Ask Eytan + + } + + fmt.Printf("%+v", vsp.Status) + + err = r.Update(context.Background(), vsp) + if err != nil { + fmt.Println(err) + } + + r.NotifyTargets(driftedTargets) + + return ctrl.Result{}, nil +} + +type InstanceTypeRange struct { + InstanceTypes []string +} + +// Returns InstanceTypes in InstanceTypeRange with types that fit the scaling policies resource requests/limits +func (r *VerticalScalingPolicyReconciler) calculateInstanceTypeRange(v *v1alpha1.VerticalScalingPolicy, instanceTypesInfo []*ec2.InstanceTypeInfo) (*InstanceTypeRange, error) { + var ( + typeRange = &InstanceTypeRange{} + hasInstanceFamily bool + resources = v.Spec.Resources + ) + + // validate provided instance family + instanceFamily, ok := v.InstanceFamily() + if ok { + if instanceFamilyExists(instanceFamily, instanceTypesInfo) { + hasInstanceFamily = true + } else { + r.Log.Info("provided instance family does not exist", "instanceFamily", instanceFamily) + } + } + + // if instance family is invalid or not provided, we need to detect it + if !hasInstanceFamily { + var err error + instanceFamily, err = r.deriveInstanceFamily(resources, instanceTypesInfo) + if err != nil { + return typeRange, errors.Wrap(err, "failed to derive instance family") + } + } + + // get types in a family that fit the requests/limits + typeRange.InstanceTypes = r.rangeInstanceTypes(resources, instanceTypesInfo, instanceFamily) + + return typeRange, nil +} + +// TODO: Alfredo + +// Decide which instance family to use +func (r *VerticalScalingPolicyReconciler) deriveInstanceFamily(resources *corev1.ResourceRequirements, instanceTypesInfo []*ec2.InstanceTypeInfo) (string, error) { + return "", nil +} + +// Returns a list of valid instance type names according to resource requirements and instance family +func (r *VerticalScalingPolicyReconciler) rangeInstanceTypes(resources *corev1.ResourceRequirements, instanceTypesInfo []*ec2.InstanceTypeInfo, family string) []string { + var computedInstances []*ec2.InstanceTypeInfo + minCPU := resources.Requests.Cpu() + maxCPU := resources.Limits.Cpu() + minMem := resources.Requests.Memory() + maxMem := resources.Limits.Memory() + + for _, instanceInfo := range instanceTypesInfo { + instanceCPUQuantity := resource.NewQuantity(*instanceInfo.VCpuInfo.DefaultCores, resource.BinarySI) + instanceMemQuantity := resource.NewQuantity(*instanceInfo.MemoryInfo.SizeInMiB, resource.BinarySI) + + // Make sure instance is in the family + if strings.HasPrefix(*instanceInfo.InstanceType, family+".") { + CPUMinHolder := instanceCPUQuantity.DeepCopy() + CPUMaxHolder := instanceCPUQuantity.DeepCopy() + MemMinHolder := instanceMemQuantity.DeepCopy() + MemMaxHolder := instanceMemQuantity.DeepCopy() + + // Subtract resource lower/upper bounds from instanceQuantity to get the difference + CPUMinHolder.Sub(*minCPU) + CPUMaxHolder.Sub(*maxCPU) + MemMinHolder.Sub(*minMem) + MemMaxHolder.Sub(*maxMem) + + CPUMinInt, _ := CPUMinHolder.AsInt64() + CPUMaxInt, _ := CPUMaxHolder.AsInt64() + MemMinInt, _ := MemMinHolder.AsInt64() + MemMaxInt, _ := MemMaxHolder.AsInt64() + + // Make sure instance CPU and Memory are greater or equal to than VSP request requirements and less than or equal to VSP limit requirements + if CPUMinInt >= 0 && CPUMaxInt >= 0 && MemMinInt >= 0 && MemMaxInt >= 0 { + computedInstances = append(computedInstances, instanceInfo) + } + } + } + + sort.Slice(computedInstances, func(i, j int) bool { + // Sort by CPU first, if they are the same then sort by Memory + if *computedInstances[i].VCpuInfo.DefaultCores != *computedInstances[j].VCpuInfo.DefaultCores { + return *computedInstances[i].VCpuInfo.DefaultCores < *computedInstances[j].VCpuInfo.DefaultCores + } + return *computedInstances[i].MemoryInfo.SizeInMiB < *computedInstances[j].MemoryInfo.SizeInMiB + }) + var computedInstancesNames []string + for _, instanceInfo := range computedInstances { + computedInstancesNames = append(computedInstancesNames, *instanceInfo.InstanceType) + } + + return computedInstancesNames +} + +func instanceFamilyExists(family string, instanceTypesInfo []*ec2.InstanceTypeInfo) bool { + families := make([]string, 0) + for _, t := range instanceTypesInfo { + instanceType := aws.StringValue(t.InstanceType) + instance := strings.Split(instanceType, ".") + families = append(families, instance[0]) + } + + return common.ContainsString(families, family) +} + +func (r *VerticalScalingPolicyReconciler) NotifyTargets(targets map[string]bool) { + for igName, _ := range targets { + r.ManagerContext.InstanceGroupEvents <- event.GenericEvent{ + Object: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + // Target name is in format my-namespace/my-ig + // TODO: use regex match to avoid index out of bounds + Namespace: strings.Split(igName, "/")[0], + Name: strings.Split(igName, "/")[1], + }, + }, + } + } +} diff --git a/go.mod b/go.mod index e9bad204..e8c3602c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/aws/aws-sdk-go v1.38.24 github.com/cucumber/godog v0.8.1 - github.com/evanphx/json-patch v4.11.0+incompatible + github.com/evanphx/json-patch v4.12.0+incompatible github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v0.4.0 github.com/keikoproj/aws-auth v0.0.0-20220330170621-38531baac0f5 @@ -15,7 +15,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.0 github.com/sirupsen/logrus v1.7.0 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect k8s.io/api v0.21.4 k8s.io/apimachinery v0.21.4 k8s.io/client-go v0.21.4 @@ -23,7 +23,7 @@ require ( ) require ( - cloud.google.com/go v0.54.0 // indirect + cloud.google.com/go v0.81.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -31,7 +31,7 @@ require ( github.com/go-logr/zapr v0.4.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-cmp v0.5.5 // indirect github.com/google/gofuzz v1.1.0 // indirect @@ -41,11 +41,11 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/json-iterator/go v1.1.11 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/karlseguin/ccache/v2 v2.0.8 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -53,22 +53,23 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 // indirect - golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 // indirect - golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect - golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect + golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect + golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect k8s.io/apiextensions-apiserver v0.21.4 // indirect k8s.io/component-base v0.21.4 // indirect - k8s.io/klog/v2 v2.8.0 // indirect - k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect - k8s.io/utils v0.0.0-20210802155522-efc7438f0176 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + k8s.io/klog/v2 v2.10.0 // indirect + k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect + k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect sigs.k8s.io/yaml v1.2.0 // indirect ) diff --git a/go.sum b/go.sum index f1e8cb3c..d9c31421 100644 --- a/go.sum +++ b/go.sum @@ -8,20 +8,35 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -71,6 +86,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -101,17 +119,23 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -131,6 +155,7 @@ github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= @@ -149,18 +174,23 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -170,6 +200,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -178,7 +209,11 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -186,11 +221,19 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -199,9 +242,11 @@ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -234,6 +279,7 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -250,8 +296,9 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -305,8 +352,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -419,8 +467,12 @@ github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlV github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -429,6 +481,9 @@ go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -473,8 +528,9 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -484,6 +540,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -500,6 +559,7 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -510,27 +570,51 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63 h1:iocB37TsdFuN6IBRZ+ry36wrkoV51/tl5vOWqkcPGvY= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -565,31 +649,52 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e h1:XMgFehsDnnLGtjvjOfqWSUzt0alpTR1RSEuznObga2c= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -632,13 +737,30 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff h1:VX/uD7MK0AHXGiScH3fsieUQUcpmRERPDYtqZdJnA+Q= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -655,12 +777,24 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -680,17 +814,48 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -702,8 +867,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -742,6 +908,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.3/go.mod h1:hUgeYHUbBp23Ue4qdX9tR8/ANi/g3ehylAqDn9NWVOg= k8s.io/api v0.21.4 h1:WtDkzTAuI31WZKDPeIYpEUA+WeUfXAmA7gwj6nzFfbc= k8s.io/api v0.21.4/go.mod h1:fTVGP+M4D8+00FN2cMnJqk/eb/GH53bvmNs2SVTmpFk= @@ -759,15 +926,19 @@ k8s.io/component-base v0.21.4 h1:Bc0AttSyhJFVXEIHz+VX+D11j/5z7SPPhl6whiXaRzs= k8s.io/component-base v0.21.4/go.mod h1:ZKG0eHVX+tUDcaoIGpU3Vtk4TIjMddN9uhEWDmW6Nyg= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= +k8s.io/klog/v2 v2.10.0 h1:R2HDMDJsHVTHA2n4RjwbeYXdOcBymXdX/JRb1v0VGhE= +k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176 h1:Mx0aa+SUAcNRQbs5jUzV8lkDlGFU8laZsY9jrcVX5SY= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -775,8 +946,9 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyz sigs.k8s.io/controller-runtime v0.9.7 h1:DlHMlAyLpgEITVvNsuZqMbf8/sJl9HirmCZIeR5H9mQ= sigs.k8s.io/controller-runtime v0.9.7/go.mod h1:nExcHcQ2zvLMeoO9K7rOesGCmgu32srN5SENvpAEbGA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/main.go b/main.go index 8388c9a0..562946ba 100644 --- a/main.go +++ b/main.go @@ -21,9 +21,10 @@ import ( "os" runt "runtime" "sync" + "time" "github.com/keikoproj/aws-sdk-go-cache/cache" - instancemgrv1alpha1 "github.com/keikoproj/instance-manager/api/v1alpha1" + "github.com/keikoproj/instance-manager/api/v1alpha1" "github.com/keikoproj/instance-manager/controllers" "github.com/keikoproj/instance-manager/controllers/common" "github.com/keikoproj/instance-manager/controllers/providers/aws" @@ -34,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics" // +kubebuilder:scaffold:imports @@ -47,7 +49,7 @@ var ( const controllerVersion = "instancemgr-0.14.1" func init() { - instancemgrv1alpha1.AddToScheme(scheme) + v1alpha1.AddToScheme(scheme) corev1.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -71,6 +73,8 @@ func main() { enableLeaderElection bool nodeRelabel bool disableWinClusterInjection bool + withVsp bool + vspResync time.Duration maxParallel int maxAPIRetries int configRetention int @@ -86,6 +90,8 @@ func main() { flag.BoolVar(&enableLeaderElection, "enable-leader-election", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") flag.BoolVar(&nodeRelabel, "node-relabel", true, "relabel nodes as they join with kubernetes.io/role label via controller") + flag.BoolVar(&withVsp, "with-vertical-scaling-policy", true, "Run the VerticalScalingPolicy controller (only supported with EKS provisioner)") + flag.DurationVar(&vspResync, "vertical-node-autoscaler-resync", time.Second*30, "the resync period for reconciling VerticalScalingPolicy objects") flag.BoolVar(&disableWinClusterInjection, "disable-windows-cluster-ca-injection", false, "Setting this to true will cause the ClusterCA and Endpoint to not be injected for Windows nodes") flag.Parse() @@ -149,6 +155,17 @@ func main() { setupLog.Info("instance-manager configmap does not exist, will not load defaults/boundaries") } + sharedContext := &controllers.SharedContext{ + Policies: make(map[string]v1alpha1.VerticalScalingPolicy), + ComputedTypes: make(map[string]string), + InstanceGroupEvents: make(chan event.GenericEvent), + InstanceGroups: make(map[string]*v1alpha1.InstanceGroup), + Nodes: make(map[string]*corev1.Node), + RWMutex: sync.RWMutex{}, + } + + withNodeWatch := nodeRelabel || withVsp + err = (&controllers.InstanceGroupReconciler{ Metrics: controllerCollector, ConfigMap: cm, @@ -157,11 +174,12 @@ func main() { ConfigNamespace: configNamespace, Namespaces: make(map[string]corev1.Namespace), NamespacesLock: &sync.RWMutex{}, - NodeRelabel: nodeRelabel, + WithNodeWatch: withNodeWatch, DisableWinClusterInjection: disableWinClusterInjection, Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName("instancegroup"), MaxParallel: maxParallel, + ManagerContext: sharedContext, Auth: &controllers.InstanceGroupAuthenticator{ Aws: awsWorker, Kubernetes: kube, @@ -171,8 +189,31 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "instancegroup") os.Exit(1) } - // +kubebuilder:scaffold:builder + if withVsp { + resyncChan := make(chan event.GenericEvent) + if err = (&controllers.VerticalScalingPolicyReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("verticalscalingpolicy"), + ManagerContext: sharedContext, + Resync: resyncChan, + Auth: &controllers.InstanceGroupAuthenticator{ + Aws: awsWorker, + Kubernetes: kube, + }, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "verticalscalingpolicy") + os.Exit(1) + } + go func() { + for range time.Tick(vspResync) { + setupLog.Info("vertical scaling policy resync") + resyncChan <- event.GenericEvent{} + } + }() + } + + // +kubebuilder:scaffold:builder setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager")