updated spark version and other tweaks

DIYBigData · Jan 31, 2022 · f43e20d · f43e20d
1 parent 87337a3
commit f43e20d
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 6 deletions.
diff --git a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml
@@ -41,7 +41,7 @@ services:
             resources:
                 limits:
                     cpus: "2.0"
-                    memory: 6g
+                    memory: 9g
     jupyter-server:
         image: master:5000/jupyter-server:latest
         hostname: jupyter-server
@@ -67,7 +67,7 @@ services:
             resources:
                 limits:
                     cpus: "2.0"
-                    memory: 6g
+                    memory: 9g
     worker-node:
         image: master:5000/worker-node:latest
         hostname: worker
@@ -91,7 +91,7 @@ services:
             mode: global
             resources:
                limits:
-                   memory: 56g
+                   memory: 52g
 networks:
     cluster_network:
         attachable: true

diff --git a/spark-qfs-swarm/jupyter-server/start-jupyter.sh b/spark-qfs-swarm/jupyter-server/start-jupyter.sh
@@ -3,7 +3,7 @@
 SHELL=/bin/bash \
     XDG_RUNTIME_DIR=/home/spark/jupyter/runtime \
     PYSPARK_DRIVER_PYTHON=jupyter \
-    PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token=''" \
+    PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token='' --NotebookApp.iopub_data_rate_limit=1.0e10" \
     $SPARK_HOME/bin/pyspark \
     --packages graphframes:graphframes:$GRAPHFRAMES_VERSION \
     --repositories https://repos.spark-packages.org/ \

diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile
@@ -16,7 +16,7 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath"
 #
 
 ARG QFS_VERSION=2.2.4
-ARG SPARK_VERSION=3.1.2
+ARG SPARK_VERSION=3.2.1
 ARG HADOOP_MINOR_VERSION=2.7
 ARG HADOOP_VERSION=2.7.2
 ARG SCALA_VERSION=2.12.13

diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf
@@ -10,10 +10,12 @@ spark.executor.cores            12
 spark.executor.extraJavaOptions -XX:+UseG1GC
 
 # driver configurations
-spark.driver.memory             8g
+spark.driver.memory             6g
+spark.driver.memoryOverhead	2g
 spark.driver.cores              2
 spark.driver.extraJavaOptions -XX:+UseG1GC
 
+
 # operational configurations
 spark.logConf                   true
 spark.worker.cleanup.enabled    true