From 7a7e798d922f270a1e471e8cd5ec1e47ada21138 Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Tue, 27 Feb 2024 12:49:00 +0545 Subject: [PATCH 1/7] Add Apache Kyuubi entrypoint in Spark rock image --- files/spark/bin/kyuubi.sh | 4 ++++ rockcraft.yaml | 45 +++++++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 files/spark/bin/kyuubi.sh diff --git a/files/spark/bin/kyuubi.sh b/files/spark/bin/kyuubi.sh new file mode 100644 index 00000000..9d21c8bd --- /dev/null +++ b/files/spark/bin/kyuubi.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +pushd /opt/kyuubi +/opt/kyuubi/bin/kyuubi start \ No newline at end of file diff --git a/rockcraft.yaml b/rockcraft.yaml index 5af9440a..b959be9c 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -48,13 +48,11 @@ services: # working-dir: /opt/spark environment: SPARK_PROPERTIES_FILE: /etc/spark8t/conf/spark-defaults.conf - thrift-server: - command: "/bin/bash /opt/pebble/thrift-server.sh" - summary: "This is the Spark Thrift Server service" + kyuubi: + command: "/bin/bash /opt/pebble/kyuubi.sh" + summary: "This is the Kyuubi service" override: replace startup: disabled - environment: - SPARK_PROPERTIES_FILE: /etc/spark8t/conf/spark-defaults.conf parts: spark: @@ -95,9 +93,41 @@ parts: - opt/spark/python - opt/spark/data + kyuubi: + plugin: dump + after: [ spark ] + source: https://dlcdn.apache.org/kyuubi/kyuubi-1.8.1/apache-kyuubi-1.8.1-bin.tgz + source-checksum: sha512/355ebbc184bbb0ab2fe521dda213a7769dae222a6a278a1fc0b3bf1173fd26091ccbc073038564ea8bb2ad2d5f5d1049c19508201ed710978d2ea79d20cd5726 + organize: + beeline-jars: opt/kyuubi/beeline-jars + bin: opt/kyuubi/bin + charts: opt/kyuubi/charts + conf: opt/kyuubi/conf + db-scripts: opt/kyuubi/db-scripts + docker: opt/kyuubi/docker + externals: opt/kyuubi/externals + jars: opt/kyuubi/jars + logs: opt/kyuubi/logs + pid: opt/kyuubi/pid + web-ui: opt/kyuubi/web-ui + work: opt/kyuubi/work + stage: + - opt/kyuubi/beeline-jars + - opt/kyuubi/bin + - opt/kyuubi/charts + - opt/kyuubi/conf + - opt/kyuubi/db-scripts + - opt/kyuubi/docker + - opt/kyuubi/externals + - opt/kyuubi/jars + - opt/kyuubi/logs + - opt/kyuubi/pid + - opt/kyuubi/web-ui + - opt/kyuubi/work + dependencies: plugin: nil - after: [ spark ] + after: [ kyuubi ] build-packages: - wget overlay-script: | @@ -237,6 +267,9 @@ parts: chown -R ${SPARK_GID}:${SPARK_UID} opt/spark chmod -R 750 opt/spark + chown -R ${SPARK_GID}:${SPARK_UID} opt/kyuubi + chmod -R 750 opt/kyuubi + mkdir -p var/lib/spark mkdir -p var/lib/spark/notebook chown -R ${SPARK_GID}:${SPARK_UID} var/lib/spark From 003b164f41a27e67c75f910f5d8417d8660eb33b Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Tue, 27 Feb 2024 17:07:04 +0545 Subject: [PATCH 2/7] Make kyuubi run as foreground process --- files/spark/bin/kyuubi.sh | 2 +- rockcraft.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/files/spark/bin/kyuubi.sh b/files/spark/bin/kyuubi.sh index 9d21c8bd..321572cd 100644 --- a/files/spark/bin/kyuubi.sh +++ b/files/spark/bin/kyuubi.sh @@ -1,4 +1,4 @@ #!/bin/bash pushd /opt/kyuubi -/opt/kyuubi/bin/kyuubi start \ No newline at end of file +/opt/kyuubi/bin/kyuubi run \ No newline at end of file diff --git a/rockcraft.yaml b/rockcraft.yaml index b959be9c..a628a659 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -219,6 +219,7 @@ parts: bin/sparkd.sh: opt/pebble/sparkd.sh bin/history-server.sh: opt/pebble/history-server.sh bin/thrift-server.sh: opt/pebble/thrift-server.sh + bin/kyuubi.sh: opt/pebble/kyuubi.sh bin/spark-client.pyspark: opt/spark-client/python/bin/spark-client.pyspark bin/spark-client.spark-sql: opt/spark-client/python/bin/spark-client.spark-sql bin/spark-client.service-account-registry: opt/spark-client/python/bin/spark-client.service-account-registry @@ -229,6 +230,7 @@ parts: - opt/pebble/sparkd.sh - opt/pebble/history-server.sh - opt/pebble/thrift-server.sh + - opt/pebble/kyuubi.sh - opt/spark-client/python/bin/spark-client.pyspark - opt/spark-client/python/bin/spark-client.spark-sql - opt/spark-client/python/bin/spark-client.service-account-registry From a147891f1265e2a9370552304ce33a74af06cfa3 Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Mon, 4 Mar 2024 16:51:45 +0545 Subject: [PATCH 3/7] Add kyuubi-env.sh to image --- files/spark/conf/kyuubi-env.sh | 2 ++ rockcraft.yaml | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 files/spark/conf/kyuubi-env.sh diff --git a/files/spark/conf/kyuubi-env.sh b/files/spark/conf/kyuubi-env.sh new file mode 100644 index 00000000..c89c8b80 --- /dev/null +++ b/files/spark/conf/kyuubi-env.sh @@ -0,0 +1,2 @@ +export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk-amd64/" +export SPARK_HOME="/opt/spark/" \ No newline at end of file diff --git a/rockcraft.yaml b/rockcraft.yaml index a628a659..2502c2e8 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -216,6 +216,8 @@ parts: source: files/spark organize: conf/spark-defaults.conf: etc/spark8t/conf/spark-defaults.conf + # conf/spark-defaults.conf: opt/spark/conf/spark-defaults.conf + conf/kyuubi-env.sh: opt/kyuubi/conf/kyuubi-env.sh bin/sparkd.sh: opt/pebble/sparkd.sh bin/history-server.sh: opt/pebble/history-server.sh bin/thrift-server.sh: opt/pebble/thrift-server.sh @@ -231,6 +233,7 @@ parts: - opt/pebble/history-server.sh - opt/pebble/thrift-server.sh - opt/pebble/kyuubi.sh + - opt/kyuubi/conf/kyuubi-env.sh - opt/spark-client/python/bin/spark-client.pyspark - opt/spark-client/python/bin/spark-client.spark-sql - opt/spark-client/python/bin/spark-client.service-account-registry From be19cfa3f0d45b6e70fbbed644e7063f71b7c5f6 Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Tue, 5 Mar 2024 09:24:13 +0545 Subject: [PATCH 4/7] Chnge the path of JAVA_HOME --- files/spark/conf/kyuubi-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/spark/conf/kyuubi-env.sh b/files/spark/conf/kyuubi-env.sh index c89c8b80..5f7b1d1f 100644 --- a/files/spark/conf/kyuubi-env.sh +++ b/files/spark/conf/kyuubi-env.sh @@ -1,2 +1,2 @@ -export JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk-amd64/" +export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" export SPARK_HOME="/opt/spark/" \ No newline at end of file From f5792e076efe8a0bd882db554f7c702d8f00e2ad Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Fri, 8 Mar 2024 09:37:35 +0545 Subject: [PATCH 5/7] Remove a commented line. --- rockcraft.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/rockcraft.yaml b/rockcraft.yaml index 2502c2e8..a1d9980e 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -216,7 +216,6 @@ parts: source: files/spark organize: conf/spark-defaults.conf: etc/spark8t/conf/spark-defaults.conf - # conf/spark-defaults.conf: opt/spark/conf/spark-defaults.conf conf/kyuubi-env.sh: opt/kyuubi/conf/kyuubi-env.sh bin/sparkd.sh: opt/pebble/sparkd.sh bin/history-server.sh: opt/pebble/history-server.sh From f1e514bdf82e928b623a25eff0e76628f18c5a6a Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Tue, 12 Mar 2024 22:57:29 +0545 Subject: [PATCH 6/7] Dump kyuubi as a whole into rock image --- rockcraft.yaml | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/rockcraft.yaml b/rockcraft.yaml index a1d9980e..65311749 100644 --- a/rockcraft.yaml +++ b/rockcraft.yaml @@ -98,32 +98,10 @@ parts: after: [ spark ] source: https://dlcdn.apache.org/kyuubi/kyuubi-1.8.1/apache-kyuubi-1.8.1-bin.tgz source-checksum: sha512/355ebbc184bbb0ab2fe521dda213a7769dae222a6a278a1fc0b3bf1173fd26091ccbc073038564ea8bb2ad2d5f5d1049c19508201ed710978d2ea79d20cd5726 - organize: - beeline-jars: opt/kyuubi/beeline-jars - bin: opt/kyuubi/bin - charts: opt/kyuubi/charts - conf: opt/kyuubi/conf - db-scripts: opt/kyuubi/db-scripts - docker: opt/kyuubi/docker - externals: opt/kyuubi/externals - jars: opt/kyuubi/jars - logs: opt/kyuubi/logs - pid: opt/kyuubi/pid - web-ui: opt/kyuubi/web-ui - work: opt/kyuubi/work + override-build: | + mkdir -p $CRAFT_PART_INSTALL/opt/kyuubi && cp -r ./* $CRAFT_PART_INSTALL/opt/kyuubi stage: - - opt/kyuubi/beeline-jars - - opt/kyuubi/bin - - opt/kyuubi/charts - - opt/kyuubi/conf - - opt/kyuubi/db-scripts - - opt/kyuubi/docker - - opt/kyuubi/externals - - opt/kyuubi/jars - - opt/kyuubi/logs - - opt/kyuubi/pid - - opt/kyuubi/web-ui - - opt/kyuubi/work + - opt/kyuubi dependencies: plugin: nil From 34f6b8b5580ad66621d44b35c272fae620c47c95 Mon Sep 17 00:00:00 2001 From: Bikalpa Dhakal Date: Wed, 13 Mar 2024 14:10:08 +0545 Subject: [PATCH 7/7] Add SPARK_CONF_DIR variable --- files/spark/conf/kyuubi-env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/files/spark/conf/kyuubi-env.sh b/files/spark/conf/kyuubi-env.sh index 5f7b1d1f..ed95fdfc 100644 --- a/files/spark/conf/kyuubi-env.sh +++ b/files/spark/conf/kyuubi-env.sh @@ -1,2 +1,3 @@ export JAVA_HOME="/usr/lib/jvm/java-11-openjdk-amd64" -export SPARK_HOME="/opt/spark/" \ No newline at end of file +export SPARK_HOME="/opt/spark/" +export SPARK_CONF_DIR="/etc/spark8t/conf"