From 0934361dbf725aead5e243672a30c508c9663e41 Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Sat, 16 Mar 2024 11:23:13 +0100
Subject: [PATCH 1/9] Add servlet filter to Charmed Spark image

---
 rockcraft.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index c73a5084..c914f6b5 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -116,6 +116,7 @@ parts:
       SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR='a351ebc4f81d20e0349b3c0f85f34f443a37ce9d'
       SHA1SUM_HADOOP_AWS_JAR='d5e162564701848b0921b80aedef9e64435333cc'
       SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='fc52ba79af46e008b1463da4c0852564f2bfce21668468b683550df1f1ff3e4f149641bbce1cffb24510569c1a441bb47f73dd2b0cff87073631c391dc248211'
+      SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
       SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
       mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
       cd $CRAFT_PART_INSTALL/opt/spark/jars
@@ -134,13 +135,19 @@ parts:
           exit 1
       fi
       wget -q "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-3.4-1.0.0/spark-metrics-assembly-3.4-1.0.0.jar"
-      wget -q "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-3.4-1.0.0/spark-metrics-assembly-3.4-1.0.0.jar.sha512"
       echo "${SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR} spark-metrics-assembly-3.4-1.0.0.jar" | sha512sum --check
       if  [[ $? -ne 0 ]]
         then
           echo "DOWNLOAD ERROR: spark-metrics-assembly-3.4-1.0.0.jar could not be downloaded properly! Exiting...." >&2
           exit 1
       fi
+      wget -q "https://github.com/canonical/central-uploader/releases/download/servlet-filters-0.0.1/servlet-filters-0.0.1.jar"
+      echo "${SHA512SUM_SPARK_SERVLET_FILTER_JAR} servlet-filters-0.0.1.jar" | sha512sum --check
+      if  [[ $? -ne 0 ]]
+        then
+          echo "DOWNLOAD ERROR: servlet-filters-0.0.1.jar could not be downloaded properly! Exiting...." >&2
+          exit 1
+      fi
       wget -q "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar"
       echo "${SHA1SUM_ICEBERG_JAR} iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" | sha1sum --check
       if  [[ $? -ne 0 ]]

From 3559d317beb2cd67c6bba97a260c2846a4696b8e Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 12:30:10 +0100
Subject: [PATCH 2/9] Update dependencies logic

---
 rockcraft.yaml | 62 +++++++++++++++++++-------------------------------
 1 file changed, 24 insertions(+), 38 deletions(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index c914f6b5..9a51c398 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -113,48 +113,34 @@ parts:
       HADOOP_AWS_VERSION='3.3.6'
       ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12'
       ICEBERG_VERSION='1.4.3'
+      SPARK_METRICS_VERSION='3.4-1.0.0'
+      SERVLET_FILTERS_VERSION='0.0.1'
       SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR='a351ebc4f81d20e0349b3c0f85f34f443a37ce9d'
       SHA1SUM_HADOOP_AWS_JAR='d5e162564701848b0921b80aedef9e64435333cc'
+      SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
       SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='fc52ba79af46e008b1463da4c0852564f2bfce21668468b683550df1f1ff3e4f149641bbce1cffb24510569c1a441bb47f73dd2b0cff87073631c391dc248211'
       SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
-      SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
-      mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
-      cd $CRAFT_PART_INSTALL/opt/spark/jars
-      wget -q "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_SDK_BUNDLE_VERSION}/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
-      echo "${SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR} aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar"
-      echo "${SHA1SUM_HADOOP_AWS_JAR}  hadoop-aws-${HADOOP_AWS_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: hadoop-aws-${HADOOP_AWS_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-3.4-1.0.0/spark-metrics-assembly-3.4-1.0.0.jar"
-      echo "${SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR} spark-metrics-assembly-3.4-1.0.0.jar" | sha512sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: spark-metrics-assembly-3.4-1.0.0.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://github.com/canonical/central-uploader/releases/download/servlet-filters-0.0.1/servlet-filters-0.0.1.jar"
-      echo "${SHA512SUM_SPARK_SERVLET_FILTER_JAR} servlet-filters-0.0.1.jar" | sha512sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: servlet-filters-0.0.1.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar"
-      echo "${SHA1SUM_ICEBERG_JAR} iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
+
+      JARS=(
+      "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/LIB_VERSION/aws-java-sdk-bundle-LIB_VERSION.jar $AWS_JAVA_SDK_BUNDLE_VERSION sha1sum $SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR"
+      "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/LIB_VERSION/hadoop-aws-LIB_VERSION.jar $HADOOP_AWS_VERSION sha1sum $SHA1SUM_HADOOP_AWS_JAR"
+      "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/LIB_VERSION/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-LIB_VERSION.jar $ICEBERG_VERSION sha1sum $SHA1SUM_ICEBERG_JAR"
+      "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR"
+      "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR"
+      )
+
+      for ENTRY in "${JARS[@]}"; do
+        echo "$ENTRY"
+        URL_BASE=$(echo "$ENTRY" | awk '{print $1}')
+        LIB_VERSION=$(echo "$ENTRY" | awk '{print $2}')
+        SHA=$(echo "$ENTRY" | awk '{print $3}')
+        EXPECTED_SHA=$(echo "$ENTRY" | awk '{print $4}')
+        URL=$(echo "$URL_BASE" | sed "s/LIB_VERSION/$LIB_VERSION/g")
+        JAR_FILE=$(echo "${URL##*/}")
+        curl -O "$URL"
+        ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
+        [ $ACTUAL_SHA != $EXPECTED_SHA ] && exit 1
+      done
     stage:
       - opt/spark/jars
 

From 5db410eca577bd2e2e8603fe2df0b6f6ae406de3 Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 13:23:33 +0100
Subject: [PATCH 3/9] Small fix

---
 rockcraft.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index 9a51c398..ccbfd70d 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -109,6 +109,9 @@ parts:
     build-packages:
       - wget
     overlay-script: |
+      mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
+      cd $CRAFT_PART_INSTALL/opt/spark/jars
+      
       AWS_JAVA_SDK_BUNDLE_VERSION='1.12.540'
       HADOOP_AWS_VERSION='3.3.6'
       ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12'

From fabe73a4c3d2600065189dd38d4e1ca2e3eccdb7 Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 13:28:26 +0100
Subject: [PATCH 4/9] Fix linting

---
 .github/workflows/build.yaml | 2 +-
 rockcraft.yaml               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 6d8e318a..bb0133a7 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -14,7 +14,7 @@ jobs:
         run: python3 -m pip install yamllint
       - name: YAML Lint
         run:  |
-          yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}}}" \
+          yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
             --no-warnings rockcraft.yaml
 
   build:
diff --git a/rockcraft.yaml b/rockcraft.yaml
index ccbfd70d..adfb3ee6 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -111,7 +111,7 @@ parts:
     overlay-script: |
       mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
       cd $CRAFT_PART_INSTALL/opt/spark/jars
-      
+
       AWS_JAVA_SDK_BUNDLE_VERSION='1.12.540'
       HADOOP_AWS_VERSION='3.3.6'
       ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12'

From 6e802fe8109b43ec1be579c8624fa73a79d69b67 Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 15:40:32 +0100
Subject: [PATCH 5/9] Change curl

---
 rockcraft.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index adfb3ee6..bbd3452e 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -116,12 +116,12 @@ parts:
       HADOOP_AWS_VERSION='3.3.6'
       ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12'
       ICEBERG_VERSION='1.4.3'
-      SPARK_METRICS_VERSION='3.4-1.0.0'
+      SPARK_METRICS_VERSION='3.4-1.0.1'
       SERVLET_FILTERS_VERSION='0.0.1'
       SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR='a351ebc4f81d20e0349b3c0f85f34f443a37ce9d'
       SHA1SUM_HADOOP_AWS_JAR='d5e162564701848b0921b80aedef9e64435333cc'
       SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
-      SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='fc52ba79af46e008b1463da4c0852564f2bfce21668468b683550df1f1ff3e4f149641bbce1cffb24510569c1a441bb47f73dd2b0cff87073631c391dc248211'
+      SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='493cf77133cbf03e96fb848121ce10ac16e6f907f595df637649b98b42118e57d6b6e1bdab71bfee3394eb369637c5b4f6b05dd8fa30a1ff6899e74069c972ce'
       SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
 
       JARS=(
@@ -140,7 +140,7 @@ parts:
         EXPECTED_SHA=$(echo "$ENTRY" | awk '{print $4}')
         URL=$(echo "$URL_BASE" | sed "s/LIB_VERSION/$LIB_VERSION/g")
         JAR_FILE=$(echo "${URL##*/}")
-        curl -O "$URL"
+        wget "$URL"
         ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
         [ $ACTUAL_SHA != $EXPECTED_SHA ] && exit 1
       done

From a72b282537690b0a711570d608d2ef46dd2e30ec Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 16:00:39 +0100
Subject: [PATCH 6/9] wip

---
 rockcraft.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index bbd3452e..a1000bd4 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -142,7 +142,9 @@ parts:
         JAR_FILE=$(echo "${URL##*/}")
         wget "$URL"
         ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
-        [ $ACTUAL_SHA != $EXPECTED_SHA ] && exit 1
+        echo "$ACTUAL_SHA"
+        echo "$EXPECTED_SHA"
+        [ $ACTUAL_SHA != $EXPECTED_SHA ] && echo "ERROR!" && exit 1
       done
     stage:
       - opt/spark/jars

From 0bf79d26e0cf8abfd5d44123811045dcff081793 Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 16:15:09 +0100
Subject: [PATCH 7/9] wip

---
 rockcraft.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index a1000bd4..533fa7f4 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -131,7 +131,6 @@ parts:
       "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR"
       "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR"
       )
-
       for ENTRY in "${JARS[@]}"; do
         echo "$ENTRY"
         URL_BASE=$(echo "$ENTRY" | awk '{print $1}')
@@ -140,11 +139,12 @@ parts:
         EXPECTED_SHA=$(echo "$ENTRY" | awk '{print $4}')
         URL=$(echo "$URL_BASE" | sed "s/LIB_VERSION/$LIB_VERSION/g")
         JAR_FILE=$(echo "${URL##*/}")
+        echo "Download from URL: $URL"
         wget "$URL"
         ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
         echo "$ACTUAL_SHA"
         echo "$EXPECTED_SHA"
-        [ $ACTUAL_SHA != $EXPECTED_SHA ] && echo "ERROR!" && exit 1
+        [ $ACTUAL_SHA != $EXPECTED_SHA ] && echo "ERROR"  && exit 1
       done
     stage:
       - opt/spark/jars

From 950bb920a4de663fb564f3fe3a8a80e97d1de45b Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Tue, 19 Mar 2024 16:23:27 +0100
Subject: [PATCH 8/9] wip

---
 rockcraft.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index 533fa7f4..bc95afeb 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -140,7 +140,7 @@ parts:
         URL=$(echo "$URL_BASE" | sed "s/LIB_VERSION/$LIB_VERSION/g")
         JAR_FILE=$(echo "${URL##*/}")
         echo "Download from URL: $URL"
-        wget "$URL"
+        wget -q "$URL"
         ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
         echo "$ACTUAL_SHA"
         echo "$EXPECTED_SHA"

From 9b42669c71f3e298c82f50acb8f1cbee0cf15ecc Mon Sep 17 00:00:00 2001
From: Paolo Sottovia <paolo.sottovia@canonical.com>
Date: Wed, 20 Mar 2024 07:42:46 +0100
Subject: [PATCH 9/9] wip

---
 rockcraft.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rockcraft.yaml b/rockcraft.yaml
index bc95afeb..186bf74e 100644
--- a/rockcraft.yaml
+++ b/rockcraft.yaml
@@ -146,6 +146,7 @@ parts:
         echo "$EXPECTED_SHA"
         [ $ACTUAL_SHA != $EXPECTED_SHA ] && echo "ERROR"  && exit 1
       done
+      echo "All dependecies downloaded!"
     stage:
       - opt/spark/jars