updated to debian 10 and java 11

spark-qfs-swarm/qfs-master/Dockerfile

@@ -9,7 +9,7 @@ FROM worker-node:latest
 
 USER root
 RUN apt-get update \
- && apt-get install -y wget vim openssh-client \
+ && apt-get install -y wget vim openssh-client python2.7-minimal \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
@@ -19,8 +19,10 @@ COPY start-qfs-master.sh /
 
 USER spark
 # create some useful bash aliases for when at bash shell prompt of this image
+#  && echo 'alias qfs="qfs -fs qfs://qfs-master:20000 -D fs.trash.minPathDepth=2 -D fs.createParams=2"' >> ~/.bash_aliases \
+
 RUN echo 'export PATH=$PATH:$QFS_HOME/bin/:$QFS_HOME/bin/tools/' >> ~/.bash_aliases \
- && echo 'alias qfs="qfs -fs qfs://qfs-master:20000 -D fs.trash.minPathDepth=2 -D fs.createParams=2"' >> ~/.bash_aliases \
+ && echo 'alias qfs="qfs -fs qfs://qfs-master:20000 -D fs.trash.minPathDepth=2"' >> ~/.bash_aliases \
  && echo 'alias cptoqfs="cptoqfs -s qfs-master -p 20000"' >> ~/.bash_aliases \
  && echo 'alias cpfromqfs="cpfromqfs -s qfs-master -p 20000"' >> ~/.bash_aliases \
  && echo 'alias qfsshell="qfsshell -s qfs-master -p 20000"' >> ~/.bash_aliases \

spark-qfs-swarm/qfs-master/start-qfs-master.sh

@@ -2,7 +2,7 @@
 
 $QFS_HOME/bin/metaserver $QFS_HOME/conf/Metaserver.prp &> $QFS_LOGS_DIR/metaserver.log &
 
-python2 $QFS_HOME/webui/qfsstatus.py $QFS_HOME/conf/webUI.cfg &> $QFS_LOGS_DIR/webui.log &
+python2.7 $QFS_HOME/webui/qfsstatus.py $QFS_HOME/conf/webUI.cfg &> $QFS_LOGS_DIR/webui.log &
 
 $QFS_HOME/bin/tools/qfs -fs qfs://qfs-master:20000 -D fs.trash.minPathDepth=2 -runEmptier &
 

spark-qfs-swarm/worker-node/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.7-stretch
+FROM python:3.10-slim-buster
 MAINTAINER Michael Kamprath "https://github.com/michaelkamprath"
 #
 # Base image for Apache Spark standalone cluster with QFS
@@ -47,7 +47,7 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # JAVA & SCALA
 RUN apt-get update \
- && apt-get install -y openjdk-8-jre \
+ && apt-get install -y openjdk-11-jre \
  && apt-get remove scala-library scala  \
  && curl -o scala-${SCALA_VERSION}.deb https://www.scala-lang.org/files/archive/scala-${SCALA_VERSION}.deb \
  && dpkg -i scala-${SCALA_VERSION}.deb \
@@ -59,11 +59,11 @@ RUN apt-get update \
 RUN useradd -m -s /bin/bash spark
 
 # QFS
-ARG QFS_PACKAGE=qfs-debian-9-${QFS_VERSION}-x86_64
+ARG QFS_PACKAGE=qfs-debian-10-${QFS_VERSION}-x86_64
 ENV QFS_HOME /usr/qfs-${QFS_VERSION}
 ENV QFS_LOGS_DIR /data/qfs/logs
 ENV LD_LIBRARY_PATH ${QFS_HOME}/lib
-ARG QFS_DOWNLOAD_URL="https://s3.amazonaws.com/quantcast-qfs/qfs-debian-9-${QFS_VERSION}-x86_64.tgz"
+ARG QFS_DOWNLOAD_URL="https://s3.amazonaws.com/quantcast-qfs/qfs-debian-10-${QFS_VERSION}-x86_64.tgz"
 RUN echo "Downloading QFS from : ${QFS_DOWNLOAD_URL}\n" \
 	&& curl -L --retry 3 -k \
 	     $QFS_DOWNLOAD_URL \

spark-qfs-swarm/worker-node/spark-conf/spark-defaults.conf

@@ -17,8 +17,9 @@ spark.driver.extraJavaOptions -XX:+UseG1GC
 
 
 # operational configurations
-spark.logConf                   true
-spark.worker.cleanup.enabled    true
+spark.logConf                                       true
+spark.worker.cleanup.enabled                        true
+spark.cleaner.referenceTracking.cleanCheckpoints    true
 
 # This setting is to tell the class loaders in Spark that they
 # only need to load the QFS access libraries once
@@ -41,4 +42,4 @@ spark.hadoop.fs.qfs.metaServerPort  20000
 # this spark.hadoop.fs.qfs.createParams	configure causes files written by Spark to
 # QFS to be 2x replicated  rather than using Reed-Solomon encoding. If you have at
 # least 9 chunkservers, remove this configuration to instead use Reed-Solomon encoding.
-spark.hadoop.fs.qfs.createParams    2
+# spark.hadoop.fs.qfs.createParams    2