Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
tlstools "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/helpers/tlstools"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
routev1 "github.com/openshift/api/route/v1"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -470,6 +471,8 @@ func HttpdDeployment(client client.Client, cr *miqv1alpha1.ManageIQ, scheme *run
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: volumeName, VolumeSource: corev1.VolumeSource{Secret: &secretVolumeSource}})
}

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -197,7 +198,7 @@ func ZookeeperService(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*corev1
return service, f
}

func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
func KafkaDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
deploymentLabels := map[string]string{
"name": "kafka",
"app": cr.Spec.AppName,
Expand Down Expand Up @@ -315,13 +316,15 @@ func KafkaDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.
},
},
}
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

return deployment, f, nil
}

func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme *runtime.Scheme) (*appsv1.Deployment, controllerutil.MutateFn, error) {
deploymentLabels := map[string]string{
"name": "zookeeper",
"app": cr.Spec.AppName,
Expand Down Expand Up @@ -398,6 +401,8 @@ func ZookeeperDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme) (*app
},
},
}
miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package miqtools

import (
miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -103,6 +104,8 @@ func NewMemcachedDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
deployment.Spec.Template.Spec.Containers[0].Env = addOrUpdateEnvVar(deployment.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{Name: "MEMCACHED_EXTRA_PARAMETERS", Value: "-Z -o ssl_chain_cert=/root/server.crt -o ssl_key=/root/server.key -p 11211"})
}

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand All @@ -14,7 +15,10 @@ import (
)

func ManageOperator(cr *miqv1alpha1.ManageIQ, client client.Client) (*appsv1.Deployment, controllerutil.MutateFn) {
deployment := operatorDeployment(cr, client)
podName := os.Getenv("POD_NAME")
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
replicaSet := miqutilsv1alpha1.FindReplicaSetByName(client, cr.Namespace, pod.ObjectMeta.OwnerReferences[0].Name)
deployment := miqutilsv1alpha1.FindDeploymentByName(client, cr.Namespace, replicaSet.ObjectMeta.OwnerReferences[0].Name)

f := func() error {
addAppLabel(cr.Spec.AppName, &deployment.ObjectMeta)
Expand Down Expand Up @@ -106,38 +110,10 @@ func ManageOperatorRoleBinding(cr *miqv1alpha1.ManageIQ, client client.Client) (
return operatorRoleBinding, f
}

func operatorPod(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.Pod {
operatorPodName := os.Getenv("POD_NAME")
podKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorPodName}
pod := &corev1.Pod{}
client.Get(context.TODO(), podKey, pod)

return pod
}

func operatorReplicaSet(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.ReplicaSet {
pod := operatorPod(cr, client)
operatorReplicaSetName := pod.ObjectMeta.OwnerReferences[0].Name
replicaSetKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorReplicaSetName}
replicaSet := &appsv1.ReplicaSet{}
client.Get(context.TODO(), replicaSetKey, replicaSet)

return replicaSet
}

func operatorDeployment(cr *miqv1alpha1.ManageIQ, client client.Client) *appsv1.Deployment {
replicaSet := operatorReplicaSet(cr, client)
operatorDeploymentName := replicaSet.ObjectMeta.OwnerReferences[0].Name
deploymentKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorDeploymentName}
deployment := &appsv1.Deployment{}
client.Get(context.TODO(), deploymentKey, deployment)

return deployment
}

func operatorServiceAccount(cr *miqv1alpha1.ManageIQ, client client.Client) *corev1.ServiceAccount {
deployment := operatorDeployment(cr, client)
operatorServiceAccountName := deployment.Spec.Template.Spec.ServiceAccountName
podName := os.Getenv("POD_NAME")
pod := miqutilsv1alpha1.FindPodByName(client, cr.Namespace, podName)
operatorServiceAccountName := pod.Spec.ServiceAccountName
serviceAccountKey := types.NamespacedName{Namespace: cr.Namespace, Name: operatorServiceAccountName}
serviceAccount := &corev1.ServiceAccount{}
client.Get(context.TODO(), serviceAccountKey, serviceAccount)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand Down Expand Up @@ -322,6 +323,8 @@ func OrchestratorDeployment(cr *miqv1alpha1.ManageIQ, scheme *runtime.Scheme, cl
}}
deployment.Spec.Template.Spec.Volumes = addOrUpdateVolume(deployment.Spec.Template.Spec.Volumes, corev1.Volume{Name: "database-secret", VolumeSource: corev1.VolumeSource{Secret: &databaseSecretVolumeSource}})

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"

miqv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1"
miqutilsv1alpha1 "github.com/ManageIQ/manageiq-pods/manageiq-operator/api/v1alpha1/miqutils"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
resource "k8s.io/apimachinery/pkg/api/resource"
Expand Down Expand Up @@ -286,6 +287,8 @@ func PostgresqlDeployment(cr *miqv1alpha1.ManageIQ, client client.Client, scheme

addInternalCertificate(cr, deployment, client, "postgresql", "/opt/app-root/src/certificates")

miqutilsv1alpha1.SetDeploymentNodeAffinity(deployment, client)

return nil
}

Expand Down
33 changes: 33 additions & 0 deletions manageiq-operator/api/v1alpha1/miqutils/find.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package miqutils

import (
"context"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func FindPodByName(client client.Client, namespace string, name string) *corev1.Pod {
podKey := types.NamespacedName{Namespace: namespace, Name: name}
pod := &corev1.Pod{}
client.Get(context.TODO(), podKey, pod)

return pod
}

func FindReplicaSetByName(client client.Client, namespace string, name string) *appsv1.ReplicaSet {
replicaSetKey := types.NamespacedName{Namespace: namespace, Name: name}
replicaSet := &appsv1.ReplicaSet{}
client.Get(context.TODO(), replicaSetKey, replicaSet)

return replicaSet
}

func FindDeploymentByName(client client.Client, namespace string, name string) *appsv1.Deployment {
deploymentKey := types.NamespacedName{Namespace: namespace, Name: name}
deployment := &appsv1.Deployment{}
client.Get(context.TODO(), deploymentKey, deployment)

return deployment
}
61 changes: 61 additions & 0 deletions manageiq-operator/api/v1alpha1/miqutils/node_affinity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package miqutils

import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"os"
"sigs.k8s.io/controller-runtime/pkg/client"
)

func OperatorNodeAffinityArchValues(deployment *appsv1.Deployment, client client.Client) []string {
podName := os.Getenv("POD_NAME")
pod := FindPodByName(client, deployment.ObjectMeta.Namespace, podName)
values := []string{"amd64"}
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added a commit to default to amd64


if pod.Spec.Affinity == nil {
// In case we don't find the operator pod (local testing) or it doesn't have affinities
return values
}

nodeSelectorTerms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms

for _, selector := range nodeSelectorTerms {
for _, matchExpression := range selector.MatchExpressions {
if matchExpression.Key == "kubernetes.io/arch" {
values = matchExpression.Values
}
}
}

return values
}

func SetDeploymentNodeAffinity(deployment *appsv1.Deployment, client client.Client) {
operatorNodeAffinityArchValues := OperatorNodeAffinityArchValues(deployment, client)
if len(operatorNodeAffinityArchValues) == 0 {
// We're running local, can't find the operator pod, or it doesn't have any affinities to use as a template. Skip it.
return
}

matchExpression := corev1.NodeSelectorRequirement{
Key: "kubernetes.io/arch",
Operator: corev1.NodeSelectorOpIn,
Values: operatorNodeAffinityArchValues,
}

matchExpressions := []corev1.NodeSelectorRequirement{matchExpression}

nodeSelectorTerm := corev1.NodeSelectorTerm{
MatchExpressions: matchExpressions,
}

nodeSelectionTerms := []corev1.NodeSelectorTerm{nodeSelectorTerm}

deployment.Spec.Template.Spec.Affinity = &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: nodeSelectionTerms,
},
},
}
}
9 changes: 9 additions & 0 deletions manageiq-operator/config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ spec:
labels:
name: manageiq-operator
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values:
- amd64
Comment on lines +20 to +23
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we also want s390x and ppc64le like in downstream?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't build those targets in upstream, so no? I'm not sure how this will work in downstream though if we have additional arches.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In downstream we have a separate setNodeAffinity function that adds the other arches to the Kafka CR

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we also want s390x and ppc64le like in downstream?

No, we only build x86_64 upstream

I'm not sure how this will work in downstream though if we have additional arches.

We don't use this file downstream. The downstream operator will deploy the manageiq operator with the correct values.

serviceAccountName: manageiq-operator
containers:
- name: manageiq-operator
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -563,7 +563,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
logger.Info("Service has been reconciled", "component", "zookeeper", "result", result)
}

kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Scheme)
kafkaDeployment, mutateFunc, err := miqtool.KafkaDeployment(cr, r.Client, r.Scheme)
if err != nil {
return err
}
Expand All @@ -574,7 +574,7 @@ func (r *ManageIQReconciler) generateKafkaResources(cr *miqv1alpha1.ManageIQ) er
logger.Info("Deployment has been reconciled", "component", "kafka", "result", result)
}

zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Scheme)
zookeeperDeployment, mutateFunc, err := miqtool.ZookeeperDeployment(cr, r.Client, r.Scheme)
if err != nil {
return err
}
Expand Down