[DPE-3373] Add servlet filters to Charmed Spark image

.github/workflows/build.yaml

@@ -14,7 +14,7 @@ jobs:
         run: python3 -m pip install yamllint
       - name: YAML Lint
         run:  |
-          yamllint -d "{extends: relaxed, rules: {line-length: {max: 200}}}" \
+          yamllint -d "{extends: relaxed, rules: {line-length: {max: 250}}}" \
             --no-warnings rockcraft.yaml
 
   build:

rockcraft.yaml

@@ -109,45 +109,44 @@ parts:
     build-packages:
       - wget
     overlay-script: |
+      mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
+      cd $CRAFT_PART_INSTALL/opt/spark/jars
+
       AWS_JAVA_SDK_BUNDLE_VERSION='1.12.540'
       HADOOP_AWS_VERSION='3.3.6'
       ICEBERG_SPARK_RUNTIME_VERSION='3.4_2.12'
       ICEBERG_VERSION='1.4.3'
+      SPARK_METRICS_VERSION='3.4-1.0.1'
+      SERVLET_FILTERS_VERSION='0.0.1'
       SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR='a351ebc4f81d20e0349b3c0f85f34f443a37ce9d'
       SHA1SUM_HADOOP_AWS_JAR='d5e162564701848b0921b80aedef9e64435333cc'
-      SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='fc52ba79af46e008b1463da4c0852564f2bfce21668468b683550df1f1ff3e4f149641bbce1cffb24510569c1a441bb47f73dd2b0cff87073631c391dc248211'
       SHA1SUM_ICEBERG_JAR='48d553e4e5496f731b9e0e6adb5bc0fd040cb0df'
-      mkdir -p $CRAFT_PART_INSTALL/opt/spark/jars
-      cd $CRAFT_PART_INSTALL/opt/spark/jars
-      wget -q "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_JAVA_SDK_BUNDLE_VERSION}/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar"
-      echo "${SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR} aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_AWS_VERSION}/hadoop-aws-${HADOOP_AWS_VERSION}.jar"
-      echo "${SHA1SUM_HADOOP_AWS_JAR}  hadoop-aws-${HADOOP_AWS_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: hadoop-aws-${HADOOP_AWS_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-3.4-1.0.0/spark-metrics-assembly-3.4-1.0.0.jar"
-      wget -q "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-3.4-1.0.0/spark-metrics-assembly-3.4-1.0.0.jar.sha512"
-      echo "${SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR} spark-metrics-assembly-3.4-1.0.0.jar" | sha512sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: spark-metrics-assembly-3.4-1.0.0.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
-      wget -q "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar"
-      echo "${SHA1SUM_ICEBERG_JAR} iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar" | sha1sum --check
-      if  [[ $? -ne 0 ]]
-        then
-          echo "DOWNLOAD ERROR: iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar could not be downloaded properly! Exiting...." >&2
-          exit 1
-      fi
+      SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR='493cf77133cbf03e96fb848121ce10ac16e6f907f595df637649b98b42118e57d6b6e1bdab71bfee3394eb369637c5b4f6b05dd8fa30a1ff6899e74069c972ce'
+      SHA512SUM_SPARK_SERVLET_FILTER_JAR='ffeb809d58ef0151d513b09d4c2bfd5cc064b0b888ca45899687aed2f42bcb1ce9834be9709290dd70bd9df84049f02cbbff6c2d5ec3c136c278c93f167c8096'
+
+      JARS=(
+      "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/LIB_VERSION/aws-java-sdk-bundle-LIB_VERSION.jar $AWS_JAVA_SDK_BUNDLE_VERSION sha1sum $SHA1SUM_AWS_JAVA_SDK_BUNDLE_JAR"
+      "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/LIB_VERSION/hadoop-aws-LIB_VERSION.jar $HADOOP_AWS_VERSION sha1sum $SHA1SUM_HADOOP_AWS_JAR"
+      "https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/LIB_VERSION/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-LIB_VERSION.jar $ICEBERG_VERSION sha1sum $SHA1SUM_ICEBERG_JAR"
+      "https://github.com/canonical/central-uploader/releases/download/spark-metrics-assembly-LIB_VERSION/spark-metrics-assembly-LIB_VERSION.jar $SPARK_METRICS_VERSION sha512sum $SHA512SUM_SPARK_METRICS_ASSEMBLY_JAR"
+      "https://github.com/canonical/central-uploader/releases/download/servlet-filters-LIB_VERSION/servlet-filters-LIB_VERSION.jar $SERVLET_FILTERS_VERSION sha512sum $SHA512SUM_SPARK_SERVLET_FILTER_JAR"
+      )
+      for ENTRY in "${JARS[@]}"; do
+        echo "$ENTRY"
+        URL_BASE=$(echo "$ENTRY" | awk '{print $1}')
+        LIB_VERSION=$(echo "$ENTRY" | awk '{print $2}')
+        SHA=$(echo "$ENTRY" | awk '{print $3}')
+        EXPECTED_SHA=$(echo "$ENTRY" | awk '{print $4}')
+        URL=$(echo "$URL_BASE" | sed "s/LIB_VERSION/$LIB_VERSION/g")
+        JAR_FILE=$(echo "${URL##*/}")
+        echo "Download from URL: $URL"
+        wget -q "$URL"
+        ACTUAL_SHA=$("$SHA" "$JAR_FILE" | awk '{print $1}')
+        echo "$ACTUAL_SHA"
+        echo "$EXPECTED_SHA"
+        [ $ACTUAL_SHA != $EXPECTED_SHA ] && echo "ERROR"  && exit 1
+      done
+      echo "All dependecies downloaded!"
     stage:
       - opt/spark/jars