tweaked memory setting

Tweaked memory setting for spark driver. Also added seaborn graphing package.
DIYBigData · Nov 25, 2019 · 56e7a12 · 56e7a12
1 parent c7abfae
commit 56e7a12
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile
@@ -87,7 +87,7 @@ RUN mkdir -p /data/spark \
 
 # add python libraries useful in PySpark
 RUN python3 -mpip install matplotlib \
-	&& pip3 install pandas 
+	&& pip3 install pandas seaborn
 
 # set up command
 COPY start-worker-node.sh /

diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf
@@ -8,7 +8,7 @@ spark.executor.memory 		24g
 spark.executor.cores		6
 
 # driver configurations
-spark.driver.memory			4g
+spark.driver.memory			8g
 spark.driver.cores			2
 
 # operational configurations