From 3b4e44c4cdd70c9cb8201855f36a15129186b5e7 Mon Sep 17 00:00:00 2001 From: Michael Kamprath Date: Thu, 18 Jun 2020 00:47:35 -0700 Subject: [PATCH] bumped versions on spark and graphframes --- spark-qfs-swarm/jupyter-server/start-jupyter.sh | 6 +++++- spark-qfs-swarm/worker-node/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spark-qfs-swarm/jupyter-server/start-jupyter.sh b/spark-qfs-swarm/jupyter-server/start-jupyter.sh index 2cc87c3..1625047 100644 --- a/spark-qfs-swarm/jupyter-server/start-jupyter.sh +++ b/spark-qfs-swarm/jupyter-server/start-jupyter.sh @@ -1,3 +1,7 @@ #!/bin/bash -SHELL=/bin/bash XDG_RUNTIME_DIR=/home/spark/jupyter/runtime PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token=''" $SPARK_HOME/bin/pyspark --packages graphframes:graphframes:0.7.0-spark2.4-s_2.11 --master spark://spark-master:7077 +SHELL=/bin/bash \ + XDG_RUNTIME_DIR=/home/spark/jupyter/runtime \ + PYSPARK_DRIVER_PYTHON=jupyter \ + PYSPARK_DRIVER_PYTHON_OPTS="notebook --no-browser --port=7777 --notebook-dir=/home/spark/jupyter/notebooks --ip=0.0.0.0 --NotebookApp.password='' --NotebookApp.token=''" \ + $SPARK_HOME/bin/pyspark --packages graphframes:graphframes:0.8.0-spark2.4-s_2.11 --master spark://spark-master:7077 diff --git a/spark-qfs-swarm/worker-node/Dockerfile b/spark-qfs-swarm/worker-node/Dockerfile index 8b947b2..95d39a4 100644 --- a/spark-qfs-swarm/worker-node/Dockerfile +++ b/spark-qfs-swarm/worker-node/Dockerfile @@ -16,7 +16,7 @@ MAINTAINER Michael Kamprath "https://github.com/michaelkamprath" # ARG QFS_VERSION=2.2.0 -ARG SPARK_VERSION=2.4.5 +ARG SPARK_VERSION=2.4.6 ARG HADOOP_MINOR_VERSION=2.7 ARG HADOOP_VERSION=2.7.2 ARG SCALA_VERSION=2.11.12