NO-JIRA: Presubmit updates for cluster-etcd-operator (#69895)

Currently, the presubmit verification jobs for PRs into the cluster-etcd-operator have 4 consistently failing blocking jobs. Three of these will need to be investigated further, because their failures appear to be related to broken configurations in the job definitions. These include the following:
- e2e-metal-assisted
- e2e-aws-cpms
- e2e-aws-ovn-scaling

More concerning is the serial job, but upon further investigation, it became clear that the serial job looks stable, but is consistently timing out before it can complete. To address this, I've added sharding to the following:
- e2e-aws-ovn-serial

I preformed these updates on openshift-4.16 and up because these are releases we're most likely to see PRs and backports to.

Author: jaypoulz

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-main.yaml

@@ -170,6 +170,7 @@ tests:
     workflow: baremetalds-e2e
 - as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -193,10 +194,12 @@ tests:
     cluster_profile: aws
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-2
     workflow: openshift-e2e-aws-etcd-scaling
@@ -242,6 +245,7 @@ tests:
       CLUSTER_AGE_STEP: "365"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws
     test:

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.16.yaml

@@ -182,6 +182,7 @@ tests:
 - always_run: false
   as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -203,10 +204,12 @@ tests:
     cluster_profile: aws
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws
     workflow: openshift-e2e-aws-etcd-scaling

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.17.yaml

@@ -182,6 +182,7 @@ tests:
 - always_run: false
   as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -203,10 +204,12 @@ tests:
     cluster_profile: aws
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-2
     workflow: openshift-e2e-aws-etcd-scaling

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.18.yaml

@@ -182,6 +182,7 @@ tests:
 - always_run: false
   as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -203,10 +204,12 @@ tests:
     cluster_profile: aws
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-etcd-scaling
@@ -255,6 +258,7 @@ tests:
       CLUSTER_AGE_STEP: "300"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws
     test:

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.19.yaml

@@ -182,6 +182,7 @@ tests:
 - always_run: false
   as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -203,10 +204,12 @@ tests:
     cluster_profile: aws-2
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-etcd-scaling
@@ -255,6 +258,7 @@ tests:
       CLUSTER_AGE_STEP: "300"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws-3
     test:

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.20.yaml

@@ -170,6 +170,7 @@ tests:
     workflow: baremetalds-e2e
 - as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -193,10 +194,12 @@ tests:
     cluster_profile: aws-2
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-etcd-scaling
@@ -242,6 +245,7 @@ tests:
       CLUSTER_AGE_STEP: "365"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws-3
     test:

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.21.yaml

@@ -171,6 +171,7 @@ tests:
     workflow: baremetalds-e2e
 - as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -194,10 +195,12 @@ tests:
     cluster_profile: aws
     workflow: openshift-perfscale-e2e-configmap-scale
 - as: e2e-aws-ovn-serial
+  shard_count: 2
   steps:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-2
     workflow: openshift-e2e-aws-etcd-scaling
@@ -243,6 +246,7 @@ tests:
       CLUSTER_AGE_STEP: "365"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws
     test:

ci-operator/config/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-release-4.22.yaml

@@ -170,6 +170,7 @@ tests:
     workflow: baremetalds-e2e
 - as: e2e-metal-assisted
   cluster: build05
+  optional: true
   steps:
     cluster_profile: packet-assisted
     workflow: assisted-ofcir-baremetal
@@ -197,6 +198,7 @@ tests:
     cluster_profile: aws-3
     workflow: openshift-e2e-aws-serial
 - as: e2e-aws-ovn-etcd-scaling
+  optional: true
   steps:
     cluster_profile: aws-2
     workflow: openshift-e2e-aws-etcd-scaling
@@ -242,6 +244,7 @@ tests:
       CLUSTER_AGE_STEP: "365"
     workflow: openshift-e2e-cert-rotation-shutdown
 - as: e2e-aws-cpms
+  optional: true
   steps:
     cluster_profile: aws
     test:

ci-operator/jobs/openshift/cluster-etcd-operator/openshift-cluster-etcd-operator-main-presubmits.yaml

@@ -304,6 +304,7 @@ presubmits:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-openshift-cluster-etcd-operator-main-e2e-aws-cpms
+    optional: true
     rerun_command: /test e2e-aws-cpms
     spec:
       containers:
@@ -668,6 +669,7 @@ presubmits:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-openshift-cluster-etcd-operator-main-e2e-aws-ovn-etcd-scaling
+    optional: true
     rerun_command: /test e2e-aws-ovn-etcd-scaling
     spec:
       containers:
@@ -732,21 +734,22 @@ presubmits:
     - ^main$
     - ^main-
     cluster: build07
-    context: ci/prow/e2e-aws-ovn-serial
+    context: ci/prow/e2e-aws-ovn-serial-1of2
     decorate: true
     labels:
       ci-operator.openshift.io/cloud: aws
       ci-operator.openshift.io/cloud-cluster-profile: aws-3
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
-    name: pull-ci-openshift-cluster-etcd-operator-main-e2e-aws-ovn-serial
-    rerun_command: /test e2e-aws-ovn-serial
+    name: pull-ci-openshift-cluster-etcd-operator-main-e2e-aws-ovn-serial-1of2
+    rerun_command: /test e2e-aws-ovn-serial-1of2
     spec:
       containers:
       - args:
         - --gcs-upload-secret=/secrets/gcs/service-account.json
         - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
         - --lease-server-credentials-file=/etc/boskos/credentials
+        - --multi-stage-param=SHARD_ARGS="--shard-count 2 --shard-id 1"
         - --report-credentials-file=/etc/report/credentials
         - --secret-dir=/secrets/ci-pull-credentials
         - --target=e2e-aws-ovn-serial
@@ -797,7 +800,80 @@ presubmits:
       - name: result-aggregator
         secret:
           secretName: result-aggregator
-    trigger: (?m)^/test( | .* )e2e-aws-ovn-serial,?($|\s.*)
+    trigger: (?m)^/test( | .* )e2e-aws-ovn-serial-1of2,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build07
+    context: ci/prow/e2e-aws-ovn-serial-2of2
+    decorate: true
+    labels:
+      ci-operator.openshift.io/cloud: aws
+      ci-operator.openshift.io/cloud-cluster-profile: aws-3
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-cluster-etcd-operator-main-e2e-aws-ovn-serial-2of2
+    rerun_command: /test e2e-aws-ovn-serial-2of2
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --multi-stage-param=SHARD_ARGS="--shard-count 2 --shard-id 2"
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=e2e-aws-ovn-serial
+        command:
+        - ci-operator
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )e2e-aws-ovn-serial-2of2,?($|\s.*)
   - agent: kubernetes
     always_run: true
     branches:
@@ -1323,6 +1399,7 @@ presubmits:
       ci.openshift.io/generator: prowgen
       pj-rehearse.openshift.io/can-be-rehearsed: "true"
     name: pull-ci-openshift-cluster-etcd-operator-main-e2e-metal-assisted
+    optional: true
     rerun_command: /test e2e-metal-assisted
     spec:
       containers: