updated spark version

spark-qfs-swarm/jupyter-server/Dockerfile

@@ -12,10 +12,10 @@ ENV SPARK_NLP_VERSION spark-nlp-spark32_2.12:3.4.4
 USER root
 RUN apt-get install -y g++
 RUN pip3 install \
-        notebook \
+        notebook==6.4.12 \
         jupyter_nbextensions_configurator \
         jupyter_contrib_nbextensions \
-        yapf wget
+        yapf wget jupyter_server
 
 COPY start-jupyter.sh /
 
@@ -26,6 +26,12 @@ RUN jupyter nbextension enable toc2/main
 RUN jupyter nbextension enable codefolding/main
 RUN jupyter nbextension enable execute_time/ExecuteTime
 
+USER root
+RUN pip3 uninstall -y traitlets
+RUN pip3 install traitlets==5.9.0
+
+
+USER spark
 RUN mkdir -p /home/spark/jupyter/runtime \
  &&	mkdir -p /home/spark/jupyter/notebooks \
  &&	mkdir -p /home/spark/jars/

spark-qfs-swarm/worker-node/Dockerfile

@@ -15,10 +15,10 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath"
 #	spark-master - the service where the spark master runs
 #
 
-ARG QFS_VERSION=2.2.5
-ARG SPARK_VERSION=3.3.2
-ARG HADOOP_MAJOR_VERSION=2
-ARG HADOOP_VERSION=2.7.2
+ARG QFS_VERSION=2.2.6
+ARG SPARK_VERSION=3.3.3
+ARG HADOOP_MAJOR_VERSION=3
+ARG HADOOP_VERSION=3.3.1
 ARG SCALA_VERSION=2.12.15
 
 RUN apt-get update \
@@ -97,7 +97,7 @@ RUN mkdir -p /data/spark \
 
 # add python libraries useful in PySpark
 RUN python3 -mpip install matplotlib \
-	&& pip3 install pandas seaborn pyarrow spark-nlp
+	&& pip3 install pandas seaborn pyarrow spark-nlp numpy==1.23.1
 
 # copy QFS and Spark configurations
 COPY ./qfs-conf/* $QFS_HOME/conf/

spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf

@@ -14,7 +14,7 @@ spark.driver.memory             10g
 spark.driver.memoryOverhead	3g
 spark.driver.cores              2
 spark.driver.extraJavaOptions -XX:+UseG1GC
-
+spark.driver.maxResultSize	2g
 
 # operational configurations
 spark.logConf                                       true