diff --git a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml index a8b3bdd..5e8ca56 100644 --- a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml +++ b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml @@ -41,7 +41,7 @@ services: resources: limits: cpus: "2.0" - memory: 6g + memory: 9g jupyter-server: image: master:5000/jupyter-server:latest hostname: jupyter-server @@ -67,7 +67,7 @@ services: resources: limits: cpus: "2.0" - memory: 6g + memory: 9g worker-node: image: master:5000/worker-node:latest hostname: worker @@ -91,7 +91,7 @@ services: mode: global resources: limits: - memory: 56g + memory: 52g networks: cluster_network: attachable: true diff --git a/spark-qfs-swarm/jupyter-server/start-jupyter.sh b/spark-qfs-swarm/jupyter-server/start-jupyter.sh index 2be5739..efe505a 100644 --- a/spark-qfs-swarm/jupyter-server/start-jupyter.sh +++ b/spark-qfs-swarm/jupyter-server/start-jupyter.sh @@ -3,7 +3,7 @@ SHELL=/bin/bash \ XDG_RUNTIME_DIR=/home/spark/jupyter/runtime \ PYSPARK_DRIVER_PYTHON=jupyter \ - PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token=''" \ + PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token='' --NotebookApp.iopub_data_rate_limit=1.0e10" \ $SPARK_HOME/bin/pyspark \ --packages graphframes:graphframes:$GRAPHFRAMES_VERSION \ --repositories https://repos.spark-packages.org/ \ diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile index 2ba9a17..092d7ff 100644 --- a/spark-qfs-swarm/worker-node/Dockerfile +++ b/spark-qfs-swarm/worker-node/Dockerfile @@ -16,7 +16,7 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath" # ARG QFS_VERSION=2.2.4 -ARG SPARK_VERSION=3.1.2 +ARG SPARK_VERSION=3.2.1 ARG HADOOP_MINOR_VERSION=2.7 ARG HADOOP_VERSION=2.7.2 ARG SCALA_VERSION=2.12.13 diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf index 70b65d1..eb89b5b 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf @@ -10,10 +10,12 @@ spark.executor.cores 12 spark.executor.extraJavaOptions -XX:+UseG1GC # driver configurations -spark.driver.memory 8g +spark.driver.memory 6g +spark.driver.memoryOverhead 2g spark.driver.cores 2 spark.driver.extraJavaOptions -XX:+UseG1GC + # operational configurations spark.logConf true spark.worker.cleanup.enabled true