From d02d641e1e91c7678f83578dd5650eba19c9da11 Mon Sep 17 00:00:00 2001
From: Bikalpa Dhakal <bikalpa.dhakal@canonical.com>
Date: Mon, 29 Jan 2024 15:32:42 +0545
Subject: [PATCH] Fix tests not passing on CI

---
 Makefile                               |  2 +-
 tests/integration/integration-tests.sh | 63 +++++++++++++++++---------
 tests/integration/setup-aws-cli.sh     |  1 +
 3 files changed, 43 insertions(+), 23 deletions(-)

diff --git a/Makefile b/Makefile
index 042a215c..4474e3d1 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ $(K8S_TAG):
 $(AWS_TAG): $(K8S_TAG)
 	@echo "=== Setting up and configure AWS CLI ==="
 	/bin/bash ./tests/integration/setup-aws-cli.sh
-	@touch $(AWS_TAG)
+	touch $(AWS_TAG)
 
 microk8s: $(K8S_TAG)
 
diff --git a/tests/integration/integration-tests.sh b/tests/integration/integration-tests.sh
index f2b8d9e4..db30cf93 100755
--- a/tests/integration/integration-tests.sh
+++ b/tests/integration/integration-tests.sh
@@ -1,4 +1,5 @@
 #!/bin/bash
+set -x
 
 get_spark_version(){
   SPARK_VERSION=$(yq '(.version)' rockcraft.yaml)
@@ -185,15 +186,22 @@ create_s3_bucket(){
   echo "Created S3 bucket ${BUCKET_NAME}"
 }
 
+delete_s3_bucket(){
+  S3_ENDPOINT=$(get_s3_endpoint)
+  BUCKET_NAME=$1
+  aws --endpoint-url "http://$S3_ENDPOINT" s3 rb "s3://$BUCKET_NAME" --force
+  echo "Deleted S3 bucket ${BUCKET_NAME}"
+}
+
 copy_file_to_s3_bucket(){
   BUCKET_NAME=$1
   FILE_PATH=$2
   BASE_NAME=$(basename "$FILE_PATH")
-  aws s3 cp $FILE_PATH s3://"$BUCKET_NAME"/"$BASE_NAME"
+  aws --endpoint-url "http://$S3_ENDPOINT" s3 cp $FILE_PATH s3://"$BUCKET_NAME"/"$BASE_NAME"
   echo "Copied file ${FILE_PATH} to S3 bucket ${BUCKET_NAME}"
 }
 
-run_iceberg_example_in_pod(){
+test_iceberg_example_in_pod(){
   create_s3_bucket spark
   copy_file_to_s3_bucket spark ./tests/integration/resources/test-iceberg.py
 
@@ -202,27 +210,38 @@ run_iceberg_example_in_pod(){
   NUM_ROWS_TO_INSERT="4"
   PREVIOUS_DRIVER_PODS_COUNT=$(kubectl get pods -n ${NAMESPACE} | grep driver | wc -l)
 
-  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" IM="$(spark_image)" \
-      /bin/bash -c 'spark-client.spark-submit' \
-      --username $UU --namespace $NN \
-      --conf spark.kubernetes.driver.request.cores=100m \
-      --conf spark.kubernetes.executor.request.cores=100m \
-      --conf spark.kubernetes.container.image=$IM \
-      --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider \
-      --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
-      --conf spark.hadoop.fs.s3a.path.style.access=true \
-      --conf spark.hadoop.fs.s3a.access.key=$(get_s3_access_key) \
-      --conf spark.hadoop.fs.s3a.secret.key=$(get_s3_secret_key) \
-      --conf spark.jars.ivy=/tmp \
-      --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
-      --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \
-      --conf spark.sql.catalog.spark_catalog.type=hive \
-      --conf spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \
-      --conf spark.sql.catalog.local.type=hadoop \
-      --conf spark.sql.catalog.local.warehouse=s3a://spark/warehouse \
-      --conf spark.sql.defaultCatalog=local   
-      s3a://spark/test-iceberg.py -n ${NUM_ROWS_TO_INSERT}
+  kubectl exec testpod -- \
+      env \
+        UU="$USERNAME" \
+        NN="$NAMESPACE" \
+        IM="$(spark_image)" \
+        NUM_ROWS="$NUM_ROWS_TO_INSERT" \
+        ACCESS_KEY="$(get_s3_access_key)" \
+        SECRET_KEY="$(get_s3_secret_key)" \
+        S3_ENDPOINT="$(get_s3_endpoint)" \
+      /bin/bash -c '\
+        spark-client.spark-submit \
+        --username $UU --namespace $NN \
+        --conf spark.kubernetes.driver.request.cores=100m \
+        --conf spark.kubernetes.executor.request.cores=100m \
+        --conf spark.kubernetes.container.image=$IM \
+        --conf spark.hadoop.fs.s3a.aws.credentials.provider=org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider \
+        --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
+        --conf spark.hadoop.fs.s3a.path.style.access=true \
+        --conf spark.hadoop.fs.s3a.endpoint=$S3_ENDPOINT \
+        --conf spark.hadoop.fs.s3a.access.key=$ACCESS_KEY \
+        --conf spark.hadoop.fs.s3a.secret.key=$SECRET_KEY \
+        --conf spark.jars.ivy=/tmp \
+        --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
+        --conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \
+        --conf spark.sql.catalog.spark_catalog.type=hive \
+        --conf spark.sql.catalog.local=org.apache.iceberg.spark.SparkCatalog \
+        --conf spark.sql.catalog.local.type=hadoop \
+        --conf spark.sql.catalog.local.warehouse=s3a://spark/warehouse \
+        --conf spark.sql.defaultCatalog=local \
+        s3a://spark/test-iceberg.py -n $NUM_ROWS'
   
+  delete_s3_bucket spark
   DRIVER_PODS_COUNT=$(kubectl get pods -n ${NAMESPACE} | grep driver | wc -l)
 
   if [[ "${PREVIOUS_DRIVER_PODS_COUNT}" == "${DRIVER_PODS_COUNT}" ]]
diff --git a/tests/integration/setup-aws-cli.sh b/tests/integration/setup-aws-cli.sh
index c56cbda4..efc24982 100644
--- a/tests/integration/setup-aws-cli.sh
+++ b/tests/integration/setup-aws-cli.sh
@@ -11,3 +11,4 @@ S3_BUCKET="spark"
 # Configure AWS CLI credentials
 aws configure set aws_access_key_id $ACCESS_KEY
 aws configure set aws_secret_access_key $SECRET_KEY
+echo "AWS CLI credentials set successfully"