update the integration tests

canonical · Jan 31, 2024 · 4b6ac12 · 4b6ac12
1 parent 7a089dd
commit 4b6ac12
Showing 1 changed file with 72 additions and 68 deletions.
diff --git a/tests/integration/integration-tests.sh b/tests/integration/integration-tests.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+NAMESPACE=tests
+
 get_spark_version(){
   SPARK_VERSION=$(yq '(.version)' rockcraft.yaml)
   echo "$SPARK_VERSION"
@@ -27,87 +29,80 @@ validate_metrics() {
   fi
 }
 
-test_restricted_account() {
-
-  kubectl config set-context spark-context --namespace=tests --cluster=prod --user=spark
-
-  run_example_job tests spark
-}
-
 setup_user() {
-  echo "setup_user() ${1} ${2} ${3}"
+  echo "setup_user() ${1} ${2}"
 
   USERNAME=$1
   NAMESPACE=$2
 
-  kubectl create namespace ${NAMESPACE}
+  kubectl -n $NAMESPACE exec testpod-admin -- env UU="$USERNAME" NN="$NAMESPACE" \
+                /bin/bash -c 'spark-client.service-account-registry create --username $UU --namespace $NN'
 
-  if [ "$#" -gt 2 ]
-  then
-    CONTEXT=$3 
-    kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" CX="$CONTEXT" \
-                  /bin/bash -c 'spark-client.service-account-registry create --context $CX --username $UU --namespace $NN' 
-  else
-    kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" \
-                  /bin/bash -c 'spark-client.service-account-registry create --username $UU --namespace $NN' 
-  fi
+  # Create the pod with the Spark service account
+  yq ea ".spec.serviceAccountName = \"${USERNAME}\"" \
+    ./tests/integration/resources/testpod.yaml | \
+    kubectl -n tests apply -f -
 
-}
+  wait_for_pod testpod $NAMESPACE
+
+  TEST_POD_TEMPLATE=$(cat tests/integration/resources/podTemplate.yaml)
 
-setup_user_admin_context() {
-  setup_user spark tests
+  kubectl -n $NAMESPACE exec testpod -- /bin/bash -c 'cp -r /opt/spark/python /var/lib/spark/'
+  kubectl -n $NAMESPACE exec testpod -- env PTEMPLATE="$TEST_POD_TEMPLATE" /bin/bash -c 'echo "$PTEMPLATE" > /etc/spark/conf/podTemplate.yaml'
 }
 
-setup_user_restricted_context() {
-  setup_user spark tests microk8s
+setup_user_context() {
+  setup_user spark $NAMESPACE
 }
 
 cleanup_user() {
   EXIT_CODE=$1
   USERNAME=$2
   NAMESPACE=$3
 
-  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" \
+  kubectl -n $NAMESPACE delete pod testpod --wait=true
+
+  kubectl -n $NAMESPACE exec testpod-admin -- env UU="$USERNAME" NN="$NAMESPACE" \
                   /bin/bash -c 'spark-client.service-account-registry delete --username $UU --namespace $NN'  
 
-  OUTPUT=$(kubectl exec testpod -- /bin/bash -c 'spark-client.service-account-registry list')
+  OUTPUT=$(kubectl -n $NAMESPACE exec testpod-admin -- /bin/bash -c 'spark-client.service-account-registry list')
 
   EXISTS=$(echo -e "$OUTPUT" | grep "$NAMESPACE:$USERNAME" | wc -l)
 
   if [ "${EXISTS}" -ne "0" ]; then
       exit 2
   fi
 
-  kubectl delete namespace ${NAMESPACE}
-
   if [ "${EXIT_CODE}" -ne "0" ]; then
       exit 1
   fi
 }
 
 cleanup_user_success() {
   echo "cleanup_user_success()......"
-  cleanup_user 0 spark tests
+  cleanup_user 0 spark $NAMESPACE
 }
 
 cleanup_user_failure() {
   echo "cleanup_user_failure()......"
-  cleanup_user 1 spark tests
+  cleanup_user 1 spark $NAMESPACE
 }
 
-setup_test_pod() {
-  kubectl apply -f ./tests/integration/resources/testpod.yaml
+wait_for_pod() {
+
+  POD=$1
+  NAMESPACE=$2
 
   SLEEP_TIME=1
   for i in {1..5}
   do
-    pod_status=$(kubectl get pod testpod | awk '{ print $3 }' | tail -n 1)
+    pod_status=$(kubectl -n ${NAMESPACE} get pod ${POD} | awk '{ print $3 }' | tail -n 1)
     echo $pod_status
-    if [ "${pod_status}" == "Running" ]
+    if [[ "${pod_status}" == "Running" ]]
     then
         echo "testpod is Running now!"
         break
-    elif [ "${i}" -le "5" ]
+    elif [[ "${i}" -le "5" ]]
     then
         echo "Waiting for the pod to come online..."
         sleep $SLEEP_TIME
@@ -117,25 +112,41 @@ setup_test_pod() {
     fi
     SLEEP_TIME=$(expr $SLEEP_TIME \* 2);
   done
+}
 
-  TEST_POD_TEMPLATE=$(cat tests/integration/resources/podTemplate.yaml)
+setup_admin_test_pod() {
+  kubectl create ns $NAMESPACE
+
+  echo "Creating admin test-pod"
+
+  # Create a pod with admin service account
+  yq ea '.spec.containers[0].env[0].name = "KUBECONFIG" | .spec.containers[0].env[0].value = "/var/lib/spark/.kube/config" | .metadata.name = "testpod-admin"' \
+    ./tests/integration/resources/testpod.yaml | \
+    kubectl -n tests apply -f -
+
+  wait_for_pod testpod-admin $NAMESPACE
 
-  kubectl exec testpod -- /bin/bash -c 'cp -r /opt/spark/python /var/lib/spark/'
-  kubectl exec testpod -- env PTEMPLATE="$TEST_POD_TEMPLATE" /bin/bash -c 'echo "$PTEMPLATE" > /etc/spark/conf/podTemplate.yaml'
+  MY_KUBE_CONFIG=$(cat /home/${USER}/.kube/config)
+
+  kubectl -n $NAMESPACE exec testpod-admin -- /bin/bash -c 'mkdir -p ~/.kube'
+  kubectl -n $NAMESPACE exec testpod-admin -- env KCONFIG="$MY_KUBE_CONFIG" /bin/bash -c 'echo "$KCONFIG" > ~/.kube/config'
 }
 
 teardown_test_pod() {
-  kubectl delete pod testpod
+  kubectl -n $NAMESPACE delete pod testpod
+  kubectl -n $NAMESPACE delete pod testpod-admin
+
+  kubectl delete namespace $NAMESPACE
 }
 
 run_example_job_in_pod() {
   SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
 
-  PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
+  PREVIOUS_JOB=$(kubectl -n $NAMESPACE get pods | grep driver | tail -n 1 | cut -d' ' -f1)
   NAMESPACE=$1
   USERNAME=$2
 
-  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+  kubectl -n $NAMESPACE exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
                   /bin/bash -c 'spark-client.spark-submit \
                   --username $UU --namespace $NN \
                   --conf spark.kubernetes.driver.request.cores=100m \
@@ -165,11 +176,11 @@ run_example_job_in_pod() {
 run_example_job_in_pod_with_pod_templates() {
   SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
 
-  PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
+  PREVIOUS_JOB=$(kubectl -n $NAMESPACE get pods | grep driver | tail -n 1 | cut -d' ' -f1)
 
   NAMESPACE=$1
   USERNAME=$2
-  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+  kubectl -n $NAMESPACE exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
                   /bin/bash -c 'spark-client.spark-submit \
                   --username $UU --namespace $NN \
                   --conf spark.kubernetes.driver.request.cores=100m \
@@ -211,7 +222,7 @@ run_example_job_in_pod_with_metrics() {
   SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
   LOG_FILE="/tmp/server.log"
   SERVER_PORT=9091
-  PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
+  PREVIOUS_JOB=$(kubectl -n $NAMESPACE get pods | grep driver | tail -n 1 | cut -d' ' -f1)
   # start simple http server
   python3 tests/integration/resources/test_web_server.py $SERVER_PORT > $LOG_FILE &
   HTTP_SERVER_PID=$!
@@ -220,7 +231,7 @@ run_example_job_in_pod_with_metrics() {
   echo "IP: $IP_ADDRESS"
   NAMESPACE=$1
   USERNAME=$2
-  kubectl exec testpod -- env PORT="$SERVER_PORT" IP="$IP_ADDRESS" UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+  kubectl -n $NAMESPACE exec testpod -- env PORT="$SERVER_PORT" IP="$IP_ADDRESS" UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
                   /bin/bash -c 'spark-client.spark-submit \
                   --username $UU --namespace $NN \
                   --conf spark.kubernetes.driver.request.cores=100m \
@@ -258,11 +269,11 @@ run_example_job_in_pod_with_metrics() {
 run_example_job_with_error_in_pod() {
   SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
 
-  PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
+  PREVIOUS_JOB=$(kubectl -n $NAMESPACE get pods | grep driver | tail -n 1 | cut -d' ' -f1)
   NAMESPACE=$1
   USERNAME=$2
 
-  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+  kubectl -n $NAMESPACE exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
                   /bin/bash -c 'spark-client.spark-submit \
                   --username $UU --namespace $NN \
                   --conf spark.kubernetes.driver.request.cores=100m \
@@ -299,21 +310,21 @@ run_example_job_with_error_in_pod() {
 }
 
 test_example_job_in_pod_with_errors() {
-  run_example_job_with_error_in_pod tests spark
+  run_example_job_with_error_in_pod $NAMESPACE spark
 }
 
 
 test_example_job_in_pod_with_templates() {
-  run_example_job_in_pod_with_pod_templates tests spark
+  run_example_job_in_pod_with_pod_templates $NAMESPACE spark
 }
 
 
 test_example_job_in_pod() {
-  run_example_job_in_pod tests spark
+  run_example_job_in_pod $NAMESPACE spark
 }
 
 test_example_job_in_pod_with_metrics() {
-  run_example_job_in_pod_with_metrics tests spark
+  run_example_job_in_pod_with_metrics $NAMESPACE spark
 }
 
 
@@ -330,7 +341,7 @@ run_spark_shell_in_pod() {
   # Sample output
   # "Pi is roughly 3.13956232343"
 
-  echo -e "$(kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" CMDS="$SPARK_SHELL_COMMANDS" IM="$(spark_image)" /bin/bash -c 'echo "$CMDS" | spark-client.spark-shell --username $UU --namespace $NN --conf spark.kubernetes.container.image=$IM')" > spark-shell.out
+  echo -e "$(kubectl -n $NAMESPACE exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" CMDS="$SPARK_SHELL_COMMANDS" IM="$(spark_image)" /bin/bash -c 'echo "$CMDS" | spark-client.spark-shell --username $UU --namespace $NN --conf spark.kubernetes.container.image=$IM')" > spark-shell.out
 
   pi=$(cat spark-shell.out  | grep "^Pi is roughly" | rev | cut -d' ' -f1 | rev | cut -c 1-3)
   echo -e "Spark-shell Pi Job Output: \n ${pi}"
@@ -339,7 +350,7 @@ run_spark_shell_in_pod() {
 }
 
 test_spark_shell_in_pod() {
-  run_spark_shell_in_pod tests spark
+  run_spark_shell_in_pod $NAMESPACE spark
 }
 
 run_pyspark_in_pod() {
@@ -354,7 +365,7 @@ run_pyspark_in_pod() {
   # Sample output
   # "Pi is roughly 3.13956232343"
 
-  echo -e "$(kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" CMDS="$PYSPARK_COMMANDS" IM="$(spark_image)" /bin/bash -c 'echo "$CMDS" | spark-client.pyspark --username $UU --namespace $NN --conf spark.kubernetes.container.image=$IM')" > pyspark.out
+  echo -e "$(kubectl -n $NAMESPACE exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" CMDS="$PYSPARK_COMMANDS" IM="$(spark_image)" /bin/bash -c 'echo "$CMDS" | spark-client.pyspark --username $UU --namespace $NN --conf spark.kubernetes.container.image=$IM')" > pyspark.out
 
   cat pyspark.out
   pi=$(cat pyspark.out  | grep "Pi is roughly" | tail -n 1 | rev | cut -d' ' -f1 | rev | cut -c 1-3)
@@ -364,14 +375,7 @@ run_pyspark_in_pod() {
 }
 
 test_pyspark_in_pod() {
-  run_pyspark_in_pod tests spark
-}
-
-test_restricted_account_in_pod() {
-
-  kubectl config set-context spark-context --namespace=tests --cluster=prod --user=spark
-
-  run_example_job_in_pod tests spark
+  run_pyspark_in_pod $NAMESPACE spark
 }
 
 cleanup_user_failure_in_pod() {
@@ -383,44 +387,44 @@ echo -e "##################################"
 echo -e "SETUP TEST POD"
 echo -e "##################################"
 
-setup_test_pod
+setup_admin_test_pod
 
 echo -e "##################################"
 echo -e "RUN EXAMPLE JOB"
 echo -e "##################################"
 
-(setup_user_admin_context && test_example_job_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_example_job_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
 
 echo -e "##################################"
 echo -e "RUN SPARK SHELL IN POD"
 echo -e "##################################"
 
-(setup_user_admin_context && test_spark_shell_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_spark_shell_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
 
 echo -e "##################################"
 echo -e "RUN PYSPARK IN POD"
 echo -e "##################################"
 
-(setup_user_admin_context && test_pyspark_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_pyspark_in_pod && cleanup_user_success) || cleanup_user_failure_in_pod
 
 echo -e "##################################"
 echo -e "RUN EXAMPLE JOB WITH POD TEMPLATE"
 echo -e "##################################"
 
-(setup_user_admin_context && test_example_job_in_pod_with_templates && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_example_job_in_pod_with_templates && cleanup_user_success) || cleanup_user_failure_in_pod
 
 echo -e "########################################"
 echo -e "RUN EXAMPLE JOB WITH PROMETHEUS METRICS"
 echo -e "########################################"
 
-(setup_user_admin_context && test_example_job_in_pod_with_metrics && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_example_job_in_pod_with_metrics && cleanup_user_success) || cleanup_user_failure_in_pod
 
 
 echo -e "########################################"
 echo -e "RUN EXAMPLE JOB WITH ERRORS"
 echo -e "########################################"
 
-(setup_user_admin_context && test_example_job_in_pod_with_errors && cleanup_user_success) || cleanup_user_failure_in_pod
+(setup_user_context && test_example_job_in_pod_with_errors && cleanup_user_success) || cleanup_user_failure_in_pod
 echo -e "##################################"
 echo -e "TEARDOWN TEST POD"
 echo -e "##################################"