update for Spark 3.3 and a few other tweaks

DIYBigData · Jul 6, 2022 · 0df7dcb · 0df7dcb
1 parent 3b891b8
commit 0df7dcb
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 9 deletions.
diff --git a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml
@@ -41,7 +41,7 @@ services:
             resources:
                 limits:
                     cpus: "2.0"
-                    memory: 9g
+                    memory: 10g
     jupyter-server:
         image: master:5000/jupyter-server:latest
         hostname: jupyter-server
@@ -67,7 +67,7 @@ services:
             resources:
                 limits:
                     cpus: "2.0"
-                    memory: 9g
+                    memory: 10g
     worker-node:
         image: master:5000/worker-node:latest
         hostname: worker
@@ -91,7 +91,7 @@ services:
             mode: global
             resources:
                limits:
-                   memory: 52g
+                   memory: 50g
 networks:
     cluster_network:
         attachable: true

diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile
@@ -16,8 +16,8 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath"
 #
 
 ARG QFS_VERSION=2.2.5
-ARG SPARK_VERSION=3.2.1
-ARG HADOOP_MINOR_VERSION=2.7
+ARG SPARK_VERSION=3.3.0
+ARG HADOOP_MAJOR_VERSION=2
 ARG HADOOP_VERSION=2.7.2
 ARG SCALA_VERSION=2.12.15
 
@@ -77,7 +77,7 @@ RUN mkdir -p /data/qfs/ \
  && chown spark -R /data/qfs
 
 # SPARK
-ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-hadoop${HADOOP_MINOR_VERSION}
+ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR_VERSION}
 ENV SPARK_HOME /usr/spark-${SPARK_VERSION}
 ENV SPARK_DIST_CLASSPATH="$QFS_HOME/lib/hadoop-$HADOOP_VERSION-qfs-$QFS_VERSION.jar:$QFS_HOME/lib/qfs-access-$QFS_VERSION.jar"
 ENV HADOOP_CONF_DIR=${SPARK_HOME}/conf/

diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf
@@ -10,8 +10,8 @@ spark.executor.cores            12
 spark.executor.extraJavaOptions -XX:+UseG1GC
 
 # driver configurations
-spark.driver.memory             6g
-spark.driver.memoryOverhead	2g
+spark.driver.memory             10g
+spark.driver.memoryOverhead	3g
 spark.driver.cores              2
 spark.driver.extraJavaOptions -XX:+UseG1GC
 

diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh b/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh
@@ -1,5 +1,5 @@
 # the total amount of memory a worker (node) can use
-SPARK_WORKER_MEMORY=55g
+SPARK_WORKER_MEMORY=50g
 
 # the total amount of cores a worker (node) can use
 SPARK_WORKER_CORES=12