From 56e7a12b7135da11fe8232e89dfd7e8b6634d435 Mon Sep 17 00:00:00 2001 From: Michael Kamprath Date: Sun, 24 Nov 2019 19:51:00 -0800 Subject: [PATCH] tweaked memory setting Tweaked memory setting for spark driver. Also added seaborn graphing package. --- spark-qfs-swarm/worker-node/Dockerfile | 2 +- spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile index 41a0ed5..5a40c82 100644 --- a/spark-qfs-swarm/worker-node/Dockerfile +++ b/spark-qfs-swarm/worker-node/Dockerfile @@ -87,7 +87,7 @@ RUN mkdir -p /data/spark \ # add python libraries useful in PySpark RUN python3 -mpip install matplotlib \ - && pip3 install pandas + && pip3 install pandas seaborn # set up command COPY start-worker-node.sh / diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf index 8cb9440..4e22020 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf @@ -8,7 +8,7 @@ spark.executor.memory 24g spark.executor.cores 6 # driver configurations -spark.driver.memory 4g +spark.driver.memory 8g spark.driver.cores 2 # operational configurations