From a956d824e42a136b697edec189c9ea4730e4b1fc Mon Sep 17 00:00:00 2001
From: Michael Kamprath <michael@kamprath.net>
Date: Wed, 5 Feb 2020 23:40:16 -0800
Subject: [PATCH] updated QFS configuration for replication

By default, QFS saves files to the cluster using Reed-Solomon encoding. If you have
a cluster with smaller than 9 chunck servers, then can be come problematice from a
file resilience perspective (RS needs 9 chubks to reliably encode a file). This change
instead make QFS write all files with a replication factor of 2, meaning only 2
nodes are needed to reliably save a file. It's wort noting that RS encoding only
requires 1.5x the file size to save the file, and file relication of 2 requires 2x.
---
 spark-qfs-swarm/qfs-master/Dockerfile                | 2 +-
 spark-qfs-swarm/worker-node/spark-conf/core-site.xml | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/spark-qfs-swarm/qfs-master/Dockerfile b/spark-qfs-swarm/qfs-master/Dockerfile
index 6ed6d44..0136d49 100644
--- a/spark-qfs-swarm/qfs-master/Dockerfile
+++ b/spark-qfs-swarm/qfs-master/Dockerfile
@@ -22,7 +22,7 @@ COPY start-qfs-master.sh /
 USER spark
 # create some useful bash aliases for when at bash shell prompt of this image
 RUN echo 'export PATH=$PATH:$QFS_HOME/bin/:$QFS_HOME/bin/tools/' >> ~/.bash_aliases \
- && echo 'alias qfs="qfs -fs qfs://qfs-master:20000"' >> ~/.bash_aliases \
+ && echo 'alias qfs="qfs -fs qfs://qfs-master:20000 -D fs.trash.minPathDepth=2 -D fs.createParams=2"' >> ~/.bash_aliases \
  && echo 'alias cptoqfs="cptoqfs -s qfs-master -p 20000"' >> ~/.bash_aliases \
  && echo 'alias cpfromqfs="cpfromqfs -s qfs-master -p 20000"' >> ~/.bash_aliases \
  && echo 'alias qfsshell="qfsshell -s qfs-master -p 20000"' >> ~/.bash_aliases \
diff --git a/spark-qfs-swarm/worker-node/spark-conf/core-site.xml b/spark-qfs-swarm/worker-node/spark-conf/core-site.xml
index 3b6d0b2..b1c7d2c 100644
--- a/spark-qfs-swarm/worker-node/spark-conf/core-site.xml
+++ b/spark-qfs-swarm/worker-node/spark-conf/core-site.xml
@@ -20,4 +20,13 @@
     <name>fs.qfs.metaServerPort</name>
     <value>20000</value>
   </property>
+  <property>
+    <!-- This parameter makes it so that all files written from spark are
+         created with a replication of 2 rather than using the defaault
+         Reed-Solomon encoding. If you have a cluster with 9 or greater
+         nodes, remove this option to favor the Reed-Solomon encoding.
+    -->
+    <name>fs.qfs.createParams</name>
+    <value>2</value>
+  </property>
 </configuration>