Skip to content

Commit

Permalink
chore(spark-history): update spark-history image to jdk 17 and spark …
Browse files Browse the repository at this point in the history
…3.5.3 (#105)
  • Loading branch information
hussein-awala authored Nov 30, 2024
1 parent 8b6be2b commit 0507c91
Showing 1 changed file with 11 additions and 7 deletions.
18 changes: 11 additions & 7 deletions docker/spark-history.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
ARG JAVA_VERSION=11
FROM openjdk:${JAVA_VERSION}-jdk AS dependencies
ARG JAVA_VERSION=17
FROM amazoncorretto:${JAVA_VERSION} AS dependencies

RUN yum install -y curl

ARG IVY_VERSION=2.5.2
RUN wget -O ivy.jar http://search.maven.org/remotecontent?filepath=org/apache/ivy/ivy/${IVY_VERSION}/ivy-${IVY_VERSION}.jar
RUN curl -L -o ivy.jar "http://search.maven.org/remotecontent?filepath=org/apache/ivy/ivy/${IVY_VERSION}/ivy-${IVY_VERSION}.jar"

ARG STORAGE_BACKEND=local
ARG HADOOP_VERSION=3.3.4
Expand All @@ -13,10 +15,12 @@ RUN /generate-dependencies.sh ${STORAGE_BACKEND}
RUN java -jar ivy.jar -ivy ivy.xml -retrieve "lib/[artifact]-[revision](-[classifier]).[ext]"


FROM openjdk:${JAVA_VERSION}-jdk
FROM amazoncorretto:${JAVA_VERSION}

RUN yum install -y curl procps tar

ARG SPARK_VERSION=3.5.1
RUN wget -qO- https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz | tar xz -C /opt \
ARG SPARK_VERSION=3.5.3
RUN curl -sL "https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz" | tar xz -C /opt \
&& ln -s /opt/spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark

COPY --from=dependencies /lib /opt/spark/jars
Expand All @@ -26,7 +30,7 @@ ENV PATH=$PATH:$SPARK_HOME/bin
ENV SPARK_NO_DAEMONIZE=false
ENV SPARK_CONF_DIR=$SPARK_HOME/conf

RUN adduser --disabled-password --gecos '' --uid 1000 spark
RUN useradd -m -u 1000 -s /usr/sbin/nologin spark
USER spark

CMD ["/opt/spark/sbin/start-history-server.sh"]

0 comments on commit 0507c91

Please sign in to comment.