diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf index 71746bd..fb4e0c6 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf @@ -1,15 +1,16 @@ # performance optimizations spark.serializer org.apache.spark.serializer.KryoSerializer spark.default.parallelism 200 -spark.sql.shuffle.partitions 400 +spark.sql.shuffle.partitions 200 # worker node / executor set up # expecting a worker with 12 cores and 56g of memory -spark.executor.memory 25g -spark.executor.cores 6 +spark.executor.memory 51g +spark.executor.cores 12 +spark.executor.extraJavaOptions -XX:+UseG1GC # driver configurations -spark.driver.memory 8g +spark.driver.memory 9g spark.driver.cores 2 # operational configurations