further tweaks

DIYBigData · Jun 30, 2020 · 3a4e539 · 3a4e539
1 parent 1573ab0
commit 3a4e539
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf
@@ -1,15 +1,16 @@
 # performance optimizations
 spark.serializer                org.apache.spark.serializer.KryoSerializer
 spark.default.parallelism       200
-spark.sql.shuffle.partitions    400
+spark.sql.shuffle.partitions    200
 
 # worker node / executor set up
 # expecting a worker with 12 cores and 56g of memory
-spark.executor.memory           25g
-spark.executor.cores            6
+spark.executor.memory           51g
+spark.executor.cores            12
+spark.executor.extraJavaOptions -XX:+UseG1GC
 
 # driver configurations
-spark.driver.memory             8g
+spark.driver.memory             9g
 spark.driver.cores              2
 
 # operational configurations