added support for spark graphframes

DIYBigData · Nov 24, 2019 · c7abfae · c7abfae
1 parent 4f20478
commit c7abfae
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/spark-qfs-swarm/jupyter-server/start-jupyter.sh b/spark-qfs-swarm/jupyter-server/start-jupyter.sh
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-SHELL=/bin/bash XDG_RUNTIME_DIR=/home/spark/jupyter/runtime PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=* --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password=''" $SPARK_HOME/bin/pyspark --master spark://spark-master:7077
+SHELL=/bin/bash XDG_RUNTIME_DIR=/home/spark/jupyter/runtime PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=* --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password=''" $SPARK_HOME/bin/pyspark --packages graphframes:graphframes:0.7.0-spark2.4-s_2.11 --master spark://spark-master:7077
diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile
@@ -38,10 +38,6 @@ RUN apt-get update \
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/*
 
-# add python libraries useful in PySpark
-RUN python3 -mpip install matplotlib \
-	&& pip3 install pandas
-
 ENV PYTHONIOENCODING UTF-8
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
@@ -89,6 +85,10 @@ COPY ./spark-conf/* $SPARK_HOME/conf/
 RUN mkdir -p /data/spark \
  && chown spark -R /data/spark
 
+# add python libraries useful in PySpark
+RUN python3 -mpip install matplotlib \
+	&& pip3 install pandas 
+
 # set up command
 COPY start-worker-node.sh /
 USER spark