feat(RHIDP-9255): Add benchmark scenario for Orchestrator plugin (#228)

* feat(RHIDP-9255): Add benchmark scenario for Orchestrator plugin

Signed-off-by: Pavel Macík <[email protected]>

* feat(scalability): Add option to always cleanup before earh iteration

Signed-off-by: Pavel Macík <[email protected]>

---------

Signed-off-by: Pavel Macík <[email protected]>

Author: pmacik

Makefile

@@ -150,7 +150,7 @@ deploy-locust: namespace
 	else \
 		echo "Helm release \"$(LOCUST_OPERATOR)\" already exists"; \
 	fi
-	kubectl wait --timeout=180s --namespace $(LOCUST_NAMESPACE) --for=condition=ready $$(kubectl get --namespace $(LOCUST_NAMESPACE) pod -o name)
+	kubectl wait --timeout=180s --namespace $(LOCUST_NAMESPACE) --for=condition=ready $$(kubectl get --namespace $(LOCUST_NAMESPACE) pod -l app.kubernetes.io/name=locust-k8s-operator -o name)
 
 ## Uninstall locust operator helm chart
 .PHONY: undeploy-locust

ci-scripts/collect-results.sh

@@ -22,6 +22,8 @@ RHDH_NAMESPACE=${RHDH_NAMESPACE:-rhdh-performance}
 ENABLE_PROFILING="${ENABLE_PROFILING:-false}"
 RHDH_INSTALL_METHOD="${RHDH_INSTALL_METHOD:-helm}"
 LOCUST_NAMESPACE="${LOCUST_NAMESPACE:-locust-operator}"
+RHDH_METRIC="${RHDH_METRIC:-true}"
+PSQL_EXPORT="${PSQL_EXPORT:-false}"
 
 cli="oc"
 clin="$cli -n $RHDH_NAMESPACE"
@@ -36,7 +38,7 @@ gather_pod_logs() {
     for pod in $pods; do
         echo "$pod"
         containers=$($cli -n "$namespace" get pod "$pod" -o json | jq -r '.spec.containers[].name')
-        if $cli -n "$namespace" get pod "$pod" -o json | jq -e '.spec.initContainers? // empty' > /dev/null; then
+        if $cli -n "$namespace" get pod "$pod" -o json | jq -e '.spec.initContainers? // empty' >/dev/null; then
             init_containers=$($cli -n "$namespace" get pod "$pod" -o json | jq -r '.spec.initContainers[].name // empty')
         else
             init_containers=""
@@ -65,6 +67,11 @@ for label in app.kubernetes.io/name=developer-hub app.kubernetes.io/name=postgre
 done
 gather_pod_logs "${ARTIFACT_DIR}/rhdh-logs" "$pods" "$RHDH_NAMESPACE"
 
+if [ "$ENABLE_ORCHESTRATOR" == "true" ]; then
+    pods=$($clin get pods -l app.kubernetes.io/component=serverless-workflow -o jsonpath='{.items[*].metadata.name}')
+    gather_pod_logs "${ARTIFACT_DIR}/workflow-logs" "$pods" "$RHDH_NAMESPACE"
+fi
+
 monitoring_collection_data=$ARTIFACT_DIR/benchmark.json
 monitoring_collection_log=$ARTIFACT_DIR/monitoring-collection.log
 monitoring_collection_dir=$ARTIFACT_DIR/monitoring-collection-raw-data-dir
@@ -112,6 +119,7 @@ try_gather_file "${TMP_DIR}/rbac-config.yaml"
 try_gather_file "${TMP_DIR}/locust-k8s-operator.values.yaml"
 try_gather_file load-test.log
 try_gather_file postgresql.log
+try_gather_dir "${TMP_DIR}/workflows"
 
 # Metrics
 PYTHON_VENV_DIR=.venv
@@ -140,6 +148,23 @@ timestamp_diff() {
     python3 -c "from datetime import datetime; st = datetime.strptime('$started', '%Y-%m-%d %H:%M:%S.%f%z'); et = datetime.strptime('$ended', '%Y-%m-%d %H:%M:%S.%f%z'); diff = et - st; print(f'{diff.total_seconds():.9f}')"
 }
 
+metrics_config_dir="${ARTIFACT_DIR}/metrics-config"
+mkdir -p "$metrics_config_dir"
+
+collect_additional_metrics() {
+    echo "$(date -u -Ins) Collecting metrics from $1"
+    status_data.py \
+        --status-data-file "$monitoring_collection_data" \
+        --additional "$1" \
+        --monitoring-start "$mstart" \
+        --monitoring-end "$mend" \
+        --monitoring-raw-data-dir "$monitoring_collection_dir" \
+        --prometheus-host "https://$mhost" \
+        --prometheus-port 443 \
+        --prometheus-token "$($cli whoami -t)" \
+        -d >>"$monitoring_collection_log" 2>&1
+}
+
 # populate phase
 if [ "$PRE_LOAD_DB" == "true" ]; then
     start_ts="$(cat "${ARTIFACT_DIR}/populate-before")"
@@ -163,7 +188,7 @@ if [ "$PRE_LOAD_DB" == "true" ]; then
     populate_catalog_started=$(cat "${ARTIFACT_DIR}/populate-catalog-before")
     populate_catalog_ended=$(cat "${ARTIFACT_DIR}/populate-catalog-after")
     populate_catalog_duration="$(timestamp_diff "$populate_catalog_started" "$populate_catalog_ended")"
-
+    echo "$(date -u -Ins) Collecting Populate phase metrics"
     status_data.py \
         --status-data-file "$monitoring_collection_data" \
         --set \
@@ -180,16 +205,20 @@ if [ "$PRE_LOAD_DB" == "true" ]; then
         measurements.timings.populate_catalog.ended="$populate_catalog_ended" \
         measurements.timings.populate_catalog.duration="$populate_catalog_duration" \
         -d >"$monitoring_collection_log" 2>&1
-    status_data.py \
-        --status-data-file "$monitoring_collection_data" \
-        --additional config/cluster_read_config.populate.yaml \
-        --monitoring-start "$mstart" \
-        --monitoring-end "$mend" \
-        --monitoring-raw-data-dir "$monitoring_collection_dir" \
-        --prometheus-host "https://$mhost" \
-        --prometheus-port 443 \
-        --prometheus-token "$($cli whoami -t)" \
-        -d >>"$monitoring_collection_log" 2>&1
+    envsubst <config/cluster_read_config.populate.yaml >"${metrics_config_dir}/cluster_read_config.populate.yaml"
+    collect_additional_metrics "${metrics_config_dir}/cluster_read_config.populate.yaml"
+    if [ "$PSQL_EXPORT" == "true" ]; then
+        echo "$(date -u -Ins) Collecting Postgresql specific metrics (populate)"
+        envsubst <config/cluster_read_config.populate.postgresql.yaml >"${metrics_config_dir}/cluster_read_config.populate.postgresql.yaml"
+        collect_additional_metrics "${metrics_config_dir}/cluster_read_config.populate.postgresql.yaml" "$monitoring_collection_data"
+    fi
+    #NodeJS specific metrics
+    if [ "$RHDH_METRIC" == "true" ]; then
+        echo "$(date -u -Ins) Collecting NodeJS specific metrics (populate)"
+        envsubst <config/cluster_read_config.populate.nodejs.yaml >"${metrics_config_dir}/cluster_read_config.populate.nodejs.yaml"
+        collect_additional_metrics "${metrics_config_dir}/cluster_read_config.populate.nodejs.yaml"
+    fi
+
 fi
 # test phase
 start_ts="$(cat "${ARTIFACT_DIR}/benchmark-before")"
@@ -201,6 +230,7 @@ mhost=$(kubectl -n openshift-monitoring get route -l app.kubernetes.io/name=than
 mversion=$(sed -n 's/^__version__ = "\(.*\)"/\1/p' "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").py")
 benchmark_started=$(cat "${ARTIFACT_DIR}/benchmark-before")
 benchmark_ended=$(cat "${ARTIFACT_DIR}/benchmark-after")
+echo "$(date -u -Ins) Collecting Test phase metrics"
 status_data.py \
     --status-data-file "$monitoring_collection_data" \
     --set \
@@ -211,28 +241,26 @@ status_data.py \
     metadata.scenario.name="$(cat "${ARTIFACT_DIR}/benchmark-scenario")" \
     metadata.scenario.version="$mversion" \
     -d >"$monitoring_collection_log" 2>&1
-status_data.py \
-    --status-data-file "$monitoring_collection_data" \
-    --additional config/cluster_read_config.test.yaml \
-    --monitoring-start "$mstart" \
-    --monitoring-end "$mend" \
-    --monitoring-raw-data-dir "$monitoring_collection_dir" \
-    --prometheus-host "https://$mhost" \
-    --prometheus-port 443 \
-    --prometheus-token "$($cli whoami -t)" \
-    -d >>"$monitoring_collection_log" 2>&1
+envsubst <config/cluster_read_config.test.yaml >"${metrics_config_dir}/cluster_read_config.test.yaml"
+collect_additional_metrics "${metrics_config_dir}/cluster_read_config.test.yaml"
 #Scenario specific metrics
-if [ -f "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").metrics.yaml" ]; then
-    status_data.py \
-        --status-data-file "$monitoring_collection_data" \
-        --additional "scenarios/$(cat "${ARTIFACT_DIR}/benchmark-scenario").metrics.yaml" \
-        --monitoring-start "$mstart" \
-        --monitoring-end "$mend" \
-        --monitoring-raw-data-dir "$monitoring_collection_dir" \
-        --prometheus-host "https://$mhost" \
-        --prometheus-port 443 \
-        --prometheus-token "$($cli whoami -t)" \
-        -d >>"$monitoring_collection_log" 2>&1
+echo "$(date -u -Ins) Collecting Scenario specific metrics"
+benchmark_scenario=$(cat "${ARTIFACT_DIR}/benchmark-scenario")
+if [ -f "scenarios/$benchmark_scenario.metrics.yaml" ]; then
+    envsubst <"scenarios/$benchmark_scenario.metrics.yaml" >"${metrics_config_dir}/$benchmark_scenario.metrics.yaml"
+    collect_additional_metrics "${metrics_config_dir}/$benchmark_scenario.metrics.yaml"
+fi
+#Postgresql specific metrics
+if [ "$PSQL_EXPORT" == "true" ]; then
+    echo "$(date -u -Ins) Collecting Postgresql specific metrics (test)"
+    envsubst <config/cluster_read_config.test.postgresql.yaml >"${metrics_config_dir}/cluster_read_config.test.postgresql.yaml"
+    collect_additional_metrics "${metrics_config_dir}/cluster_read_config.test.postgresql.yaml"
+fi
+#NodeJS specific metrics
+if [ "$RHDH_METRIC" == "true" ]; then
+    echo "$(date -u -Ins) Collecting NodeJS specific metrics (test)"
+    envsubst <config/cluster_read_config.test.nodejs.yaml >"${metrics_config_dir}/cluster_read_config.test.nodejs.yaml"
+    collect_additional_metrics "${metrics_config_dir}/cluster_read_config.test.nodejs.yaml"
 fi
 set +u
 deactivate
@@ -254,8 +282,10 @@ if [ "$RHDH_INSTALL_METHOD" == "helm" ] && ${ENABLE_PROFILING}; then
     $clin exec "$pod" -c backstage-backend -- /bin/bash -c 'find /opt/app-root/src -name "*.heapsnapshot" -exec base64 -w0 {} \;' | base64 -d >"$memory_profile_file"
 fi
 
+echo "$(date -u -Ins) Generating summary CSV"
 ./ci-scripts/runs-to-csv.sh "$ARTIFACT_DIR" >"$ARTIFACT_DIR/summary.csv"
 
+echo "$(date -u -Ins) Collecting error reports"
 # Error report
 find "$ARTIFACT_DIR" -name load-test.log -print0 | sort -V | while IFS= read -r file; do
     if grep "Error report" "$file" >/dev/null; then

ci-scripts/rhdh-setup/create_resource.sh

@@ -395,12 +395,12 @@ create_groups() {
   if [[ "$RBAC_POLICY" == "$RBAC_POLICY_NESTED_GROUPS" ]]; then
     N="${RBAC_POLICY_SIZE:-$GROUP_COUNT}"
     [ "$N" -gt "$GROUP_COUNT" ] && N="$GROUP_COUNT"
-    seq 1 "$N" | xargs -n1 -P1 -I{} bash -lc "create_group \"\$1\"" _ {}
+    seq 1 "$N" | xargs -P1 -I{} bash -lc "create_group \"\$1\"" _ {}
     if [ "$GROUP_COUNT" -gt "$N" ]; then
-      seq $((N+1)) "$GROUP_COUNT" | xargs -n1 -P"${POPULATION_CONCURRENCY}" -I{} bash -lc "create_group \"\$1\"" _ {}
+      seq $((N+1)) "$GROUP_COUNT" | xargs -P"${POPULATION_CONCURRENCY}" -I{} bash -lc "create_group \"\$1\"" _ {}
     fi
   else
-    seq 1 "$GROUP_COUNT" | xargs -n1 -P"${POPULATION_CONCURRENCY}" -I{} bash -lc "create_group \"\$1\"" _ {}
+    seq 1 "$GROUP_COUNT" | xargs -P"${POPULATION_CONCURRENCY}" -I{} bash -lc "create_group \"\$1\"" _ {}
   fi
 }
 

ci-scripts/rhdh-setup/deploy.sh

@@ -221,6 +221,10 @@ install() {
         log_error "Installation failed!!!"
         return "$exit_code"
     fi
+
+    if ${ENABLE_ORCHESTRATOR}; then
+        install_workflows
+    fi
     psql_debug
 }
 
@@ -380,6 +384,16 @@ backstage_install() {
     done
 }
 
+install_workflows() {
+    log_info "Installing Orchestrator workflows"
+    mkdir -p "$TMP_DIR/workflows"
+    find template/workflows/basic -type f -print0 | while IFS= read -r -d '' i; do
+        # shellcheck disable=SC2094
+        envsubst <"$i" >"$TMP_DIR/workflows/$(basename "$i")"
+        $clin apply -f "$TMP_DIR/workflows/$(basename "$i")"
+    done
+}
+
 # shellcheck disable=SC2016,SC1004
 install_rhdh_with_helm() {
     chart_values=template/backstage/helm/chart-values.yaml
@@ -617,28 +631,8 @@ setup_monitoring() {
         oc -n openshift-user-workload-monitoring rollout status statefulset/prometheus-user-workload -w
     fi
 
-    log_info "Setup monitoring"
-    cat <<EOF | kubectl -n locust-operator apply -f -
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  labels:
-    app: locust-operator
-  annotations:
-    networkoperator.openshift.io/ignore-errors: ""
-  name: locust-operator-monitor
-  namespace: locust-operator
-spec:
-  endpoints:
-    - interval: 10s
-      port: prometheus-metrics
-      honorLabels: true
-  jobLabel: app
-  namespaceSelector:
-    matchNames:
-      - locust-operator
-  selector: {}
-EOF
+    log_info "Setting up Locust monitoring"
+    envsubst <template/locust-metrics/locust-service-monitor.yaml | kubectl -n "${LOCUST_NAMESPACE}" apply -f -
 }
 
 delete() {
@@ -748,7 +742,7 @@ delete_orchestrator_infra() {
     $cli delete ns knative-serving-ingress --ignore-not-found=true --wait
 }
 
-while getopts "oi:mrd" flag; do
+while getopts "oi:mrdw" flag; do
     case "${flag}" in
     o)
         export INSTALL_METHOD=olm
@@ -764,6 +758,9 @@ while getopts "oi:mrd" flag; do
         AUTH_PROVIDER="$OPTARG"
         install
         ;;
+    w)
+        install_workflows
+        ;;
     m)
         setup_monitoring
         ;;

ci-scripts/rhdh-setup/template/backstage/app-config.yaml

@@ -23,7 +23,7 @@ catalog:
         groupQuerySize: 1000
         schedule:
           frequency: 
-            minutes: 30
+            hours: 2
           timeout: 
             minutes: 1
           initialDelay: 

ci-scripts/rhdh-setup/template/backstage/helm/chart-values.yaml

@@ -25,16 +25,16 @@ orchestrator:
 upstream:
   backstage:
     appConfig:
-        database:
-          connection:
-            password: "${POSTGRESQL_ADMIN_PASSWORD}"
-            user: postgres
-        auth:
-          externalAccess:
-            - type: legacy
-              options:
-                secret: ${BACKEND_SECRET}
-                subject: legacy-catalog
+      database:
+        connection:
+          password: "${POSTGRESQL_ADMIN_PASSWORD}"
+          user: postgres
+      auth:
+        externalAccess:
+          - type: legacy
+            options:
+              secret: ${BACKEND_SECRET}
+              subject: legacy-catalog
     containerPorts:
       backend: 7007
     extraAppConfig:

ci-scripts/rhdh-setup/template/backstage/helm/orchestrator-rbac-patch.yaml

@@ -1,11 +1,4 @@
-    p, role:default/a, orchestrator.workflow.read, read, allow
-    p, role:default/a, orchestrator.workflow.execute, create, allow
     p, role:default/a, orchestrator.workflow, read, allow
-    p, role:default/a, orchestrator.workflow, use, allow
-    p, role:default/a, orchestrator.workflow.instance.read, read, allow
-    p, role:default/a, orchestrator.workflow.instance.abort, delete, allow
-    p, role:default/a, orchestrator.workflow.instances.read, read, allow
-    p, role:default/a, scaffolder.action.execute, use, allow
-    p, role:default/a, scaffolder.task.create, create, allow
-    p, role:default/a, scaffolder.task.read, read, allow
-    p, role:default/a, scaffolder.task.cancel, delete, allow
+    p, role:default/a, orchestrator.workflow.use, update, allow
+    p, role:default/a, orchestrator.workflowAdminView, read, allow
+    p, role:default/a, orchestrator.instanceAdminView, read, allow

ci-scripts/rhdh-setup/template/locust-metrics/locust-service-monitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  annotations:
+    networkoperator.openshift.io/ignore-errors: ''
+  labels:
+    app: locust-operator
+  name: ${LOCUST_NAMESPACE}-monitor
+  namespace: ${LOCUST_NAMESPACE}
+spec:
+  endpoints:
+    - honorLabels: true
+      interval: 10s
+      port: prometheus-metrics
+  jobLabel: app
+  namespaceSelector:
+    matchNames:
+      - ${LOCUST_NAMESPACE}
+  selector: {}

ci-scripts/rhdh-setup/template/workflows/basic/01-configmap_basic-props.yaml

@@ -0,0 +1,24 @@
+apiVersion: v1
+data:
+  application.properties: |2
+
+    # This property is used to select the log level, which controls the amount
+    # of information logged on HTTP requests based on the severity of the events.
+    # Possible values: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL.
+    # and see https://quarkus.io/guides/logging for documentation
+    quarkus.log.category."org.apache.http".level=INFO
+    quarkus.log.level=INFO
+
+    # enable persistence
+    kie.flyway.enabled = true
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  labels:
+    app: basic
+    app.kubernetes.io/component: serverless-workflow
+    app.kubernetes.io/managed-by: sonataflow-operator
+    app.kubernetes.io/name: basic
+    sonataflow.org/workflow-app: basic
+    sonataflow.org/workflow-namespace: ""
+  name: basic-props