diff --git a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml index 5e8ca56..c2414e2 100644 --- a/spark-qfs-swarm/deploy-spark-qfs-swarm.yml +++ b/spark-qfs-swarm/deploy-spark-qfs-swarm.yml @@ -41,7 +41,7 @@ services: resources: limits: cpus: "2.0" - memory: 9g + memory: 10g jupyter-server: image: master:5000/jupyter-server:latest hostname: jupyter-server @@ -67,7 +67,7 @@ services: resources: limits: cpus: "2.0" - memory: 9g + memory: 10g worker-node: image: master:5000/worker-node:latest hostname: worker @@ -91,7 +91,7 @@ services: mode: global resources: limits: - memory: 52g + memory: 50g networks: cluster_network: attachable: true diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile index df223cd..77e01dc 100644 --- a/spark-qfs-swarm/worker-node/Dockerfile +++ b/spark-qfs-swarm/worker-node/Dockerfile @@ -16,8 +16,8 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath" # ARG QFS_VERSION=2.2.5 -ARG SPARK_VERSION=3.2.1 -ARG HADOOP_MINOR_VERSION=2.7 +ARG SPARK_VERSION=3.3.0 +ARG HADOOP_MAJOR_VERSION=2 ARG HADOOP_VERSION=2.7.2 ARG SCALA_VERSION=2.12.15 @@ -77,7 +77,7 @@ RUN mkdir -p /data/qfs/ \ && chown spark -R /data/qfs # SPARK -ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-hadoop${HADOOP_MINOR_VERSION} +ENV SPARK_PACKAGE spark-${SPARK_VERSION}-bin-hadoop${HADOOP_MAJOR_VERSION} ENV SPARK_HOME /usr/spark-${SPARK_VERSION} ENV SPARK_DIST_CLASSPATH="$QFS_HOME/lib/hadoop-$HADOOP_VERSION-qfs-$QFS_VERSION.jar:$QFS_HOME/lib/qfs-access-$QFS_VERSION.jar" ENV HADOOP_CONF_DIR=${SPARK_HOME}/conf/ diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf index 98c9d7b..aacb301 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf @@ -10,8 +10,8 @@ spark.executor.cores 12 spark.executor.extraJavaOptions -XX:+UseG1GC # driver configurations -spark.driver.memory 6g -spark.driver.memoryOverhead 2g +spark.driver.memory 10g +spark.driver.memoryOverhead 3g spark.driver.cores 2 spark.driver.extraJavaOptions -XX:+UseG1GC diff --git a/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh b/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh index 32132f2..8984e7c 100644 --- a/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh +++ b/spark-qfs-swarm/worker-node/spark-conf/spark-env.sh @@ -1,5 +1,5 @@ # the total amount of memory a worker (node) can use -SPARK_WORKER_MEMORY=55g +SPARK_WORKER_MEMORY=50g # the total amount of cores a worker (node) can use SPARK_WORKER_CORES=12