bumpped version and tuned memory usage

DIYBigData · Sep 5, 2021 · 87337a3 · 87337a3
1 parent 0a2c693
commit 87337a3
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile
@@ -15,8 +15,8 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath"
 #	spark-master - the service where the spark master runs
 #
 
-ARG QFS_VERSION=2.2.3
-ARG SPARK_VERSION=3.1.1
+ARG QFS_VERSION=2.2.4
+ARG SPARK_VERSION=3.1.2
 ARG HADOOP_MINOR_VERSION=2.7
 ARG HADOOP_VERSION=2.7.2
 ARG SCALA_VERSION=2.12.13

diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf
@@ -5,13 +5,14 @@ spark.sql.shuffle.partitions    200
 
 # worker node / executor set up
 # expecting a worker with 12 cores and 56g of memory
-spark.executor.memory           51g
+spark.executor.memory           50g
 spark.executor.cores            12
 spark.executor.extraJavaOptions -XX:+UseG1GC
 
 # driver configurations
-spark.driver.memory             5g
+spark.driver.memory             8g
 spark.driver.cores              2
+spark.driver.extraJavaOptions -XX:+UseG1GC
 
 # operational configurations
 spark.logConf                   true