diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf index 4e22020..5085801 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf @@ -1,5 +1,5 @@ # performance optimizations -spark.serializer org.apache.spark.serializer.KryoSerializer +spark.serializer org.apache.spark.serializer.KryoSerializer spark.default.parallelism 100 # worker node / executor set up @@ -8,12 +8,18 @@ spark.executor.memory 24g spark.executor.cores 6 # driver configurations -spark.driver.memory 8g -spark.driver.cores 2 +spark.driver.memory 8g +spark.driver.cores 2 # operational configurations -spark.logConf true +spark.logConf true # This setting is to tell the class loaders in Spark that they # only need to load the QFS access libraries once spark.sql.hive.metastore.sharedPrefixes com.quantcast.qfs + +# Set up retention of Spark events to enable the history server. +# The configured directory needs to be created prior to launching +# Spark master. +spark.eventLog.enabled true +spark.eventLog.dir qfs:///history/spark/