[MISC] Fix tag version for GPU image (#102)

Author: deusebio

.github/workflows/build_gpu.yaml

@@ -37,12 +37,12 @@ jobs:
           sudo snap install rockcraft --classic --edge    
       
       - name: Build image (GPU)
-        run: sudo make build FLAVOUR=spark-gpu
+        run: sudo make build FLAVOUR=gpu
 
       - name: Get Artifact Name
         id: artifact
         run: |
-          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          GPU_ARTIFACT=$(make help FLAVOUR=gpu | grep 'Artifact: ')
           echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Change artifact permissions

.github/workflows/integration-gpu.yaml

@@ -69,7 +69,7 @@ jobs:
     - name: Get Artifact Name
       id: artifact
       run: |
-        GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+        GPU_ARTIFACT=$(make help FLAVOUR=gpu | grep 'Artifact: ')
         echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
     - name: Download artifact
@@ -84,7 +84,7 @@ jobs:
         mv charmed-spark/${{ steps.artifact.outputs.gpu_artifact_name }} .
         touch .make_cache/k8s.tag
         # Import artifact into microk8s to be used in integration tests
-        sudo make microk8s-import FLAVOUR=spark-gpu PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
+        sudo make microk8s-import FLAVOUR=gpu PREFIX=test- REPOSITORY=ghcr.io/canonical/ \
           -o ${{ steps.artifact.outputs.gpu_artifact_name }}
 
     - name: Run integration tests on GPU

.github/workflows/integration.yaml

@@ -36,7 +36,7 @@ jobs:
           echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
           KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
           echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
-          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          GPU_ARTIFACT=$(make help FLAVOUR=gpu | grep 'Artifact: ')
           echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Install and configure microk8s

.github/workflows/publish.yaml

@@ -78,7 +78,7 @@ jobs:
           echo "jupyter_artifact_name=${JUPYTER_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
           KYUUBI_ARTIFACT=$(make help FLAVOUR=kyuubi | grep 'Artifact: ')
           echo "kyuubi_artifact_name=${KYUUBI_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
-          GPU_ARTIFACT=$(make help FLAVOUR=spark-gpu | grep 'Artifact: ')
+          GPU_ARTIFACT=$(make help FLAVOUR=gpu | grep 'Artifact: ')
           echo "gpu_artifact_name=${GPU_ARTIFACT#'Artifact: '}" >> $GITHUB_OUTPUT
 
       - name: Download artifact
@@ -213,11 +213,11 @@ jobs:
           TRACK=${{ needs.release_checks.outputs.track }}
           if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi
       
-          IMAGE_NAME=$(make help REPOSITORY=${REPOSITORY} TAG=${TAG} FLAVOUR=spark-gpu help | grep "Image\:" | cut -d ":" -f2  | xargs)
+          IMAGE_NAME=$(make help REPOSITORY=${REPOSITORY} TAG=${TAG} FLAVOUR=gpu help | grep "Image\:" | cut -d ":" -f2  | xargs)
 
           # Import artifact into docker with new tag
           sudo make docker-import \
-            FLAVOUR=spark-gpu \
+            FLAVOUR=gpu \
             REPOSITORY=${REPOSITORY} \
             TAG=${TAG} \
             -o ${{ steps.artifact.outputs.gpu_artifact_name }}
@@ -232,7 +232,7 @@ jobs:
           docker push ${IMAGE_NAME}:${TAG}
 
           if [[ "$RISK" == "edge" ]]; then
-            VERSION_LONG=$(make help FLAVOUR=spark-gpu | grep "Tag\:" | cut -d ":" -f2  | xargs)
+            VERSION_LONG=$(make help FLAVOUR=gpu | grep "Tag\:" | cut -d ":" -f2  | xargs)
             VERSION_TAG="${VERSION_LONG}-${{ needs.release_checks.outputs.base }}_edge"
 
             docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG}

Makefile

@@ -54,6 +54,7 @@ JUPYTER_VERSION=$(shell cat images/metadata.yaml | yq .flavours.jupyter.version)
 
 # eg, charmed-spark-gpu
 ROCK_NAME_GPU := $(shell cat images/charmed-spark-gpu/rockcraft.yaml | yq .name)
+GPU_VERSION=$(shell cat images/metadata.yaml | yq .flavours.gpu.version)
 
 # The filename of the Rock file built during the build process.
 # eg, charmed-spark_3.5.1_amd64.rock
@@ -93,7 +94,7 @@ SPARK_GPU_ARTIFACT=$(ROCK_NAME_GPU)_$(SPARK_VERSION)_$(PLATFORM).tar
 # or,
 # ARTIFACT = "charmed-spark-kyuubi_3.5.1_amd64.tar"		TAG = "3.5.1-1.9.0"		DISPLAY_NAME = "ghcr.io/canonical/charmed-spark-kyuubi"
 # or, 
-# ARTIFACT = "charmed-spark_gpu_3.5.1_amd64.tar" 			TAG = "3.5.1"			DISPLAY_NAME = "ghcr.io/canonical/charmed-spark-gpu"
+# ARTIFACT = "charmed-spark_gpu_3.5.1_amd64.tar" 		TAG = "3.5.1-24.04.1"			DISPLAY_NAME = "ghcr.io/canonical/charmed-spark-gpu"
 
 ifeq ($(FLAVOUR), jupyter)
 	DISPLAY_NAME=$(REPOSITORY)$(PREFIX)$(ROCK_NAME)-jupyterlab
@@ -103,9 +104,9 @@ else ifeq ($(FLAVOUR), kyuubi)
 	DISPLAY_NAME=$(REPOSITORY)$(PREFIX)$(ROCK_NAME)-kyuubi
 	TAG=$(SPARK_VERSION)-$(KYUUBI_VERSION)
 	ARTIFACT=$(KYUUBI_ARTIFACT)
-else ifeq ($(FLAVOUR), spark-gpu)
+else ifeq ($(FLAVOUR), gpu)
 	DISPLAY_NAME=$(REPOSITORY)$(PREFIX)$(ROCK_NAME)-gpu
-	TAG=$(SPARK_VERSION)
+	TAG=$(SPARK_VERSION)-$(GPU_VERSION)
 	ARTIFACT=$(SPARK_GPU_ARTIFACT)
 else
 	DISPLAY_NAME=$(REPOSITORY)$(PREFIX)$(ROCK_NAME)
@@ -122,7 +123,7 @@ endif
 # K8S_MARKER: The MicroK8s cluster has been installed and configured successfully
 # AWS_MARKER: The AWS CLI has been installed and configured with valid S3 credentials from MinIO
 SPARK_MARKER=$(_MAKE_DIR)/spark-$(SPARK_VERSION).tag
-SPARK_GPU_MARKER=$(_MAKE_DIR)/spark-gpu-$(SPARK_VERSION).tag
+SPARK_GPU_MARKER=$(_MAKE_DIR)/gpu-$(SPARK_VERSION).tag
 JUPYTER_MARKER=$(_MAKE_DIR)/jupyter-$(JUPYTER_VERSION).tag
 KYUUBI_MARKER=$(_MAKE_DIR)/kyuubi-$(KYUUBI_VERSION).tag
 K8S_MARKER=$(_MAKE_DIR)/k8s.tag
@@ -174,7 +175,7 @@ help:
 
 # Recipe for creating a rock image from the current repository.
 # 
-# ROCK_FILE => charmed-spark_3.5.1_amd64.rock 
+# ROCK_FILE => charmed-spark_3.5.1_amd64.rock
 #
 $(ROCK_FILE): images/charmed-spark/rockcraft.yaml $(wildcard images/charmed-spark/*/*)
 	@echo "=== Building Charmed Image ==="
@@ -259,15 +260,15 @@ $(SPARK_GPU_MARKER): $(ROCK_FILE_GPU) images/charmed-spark-gpu/Dockerfile
 	touch $(SPARK_GPU_MARKER)
 
 # Shorthand recipe for building Spark-gpu image
-spark-gpu: $(SPARK_GPU_MARKER)
+gpu: $(SPARK_GPU_MARKER)
 
 $(ARTIFACT):
 ifeq ($(FLAVOUR), jupyter)
 	make jupyter
 else ifeq ($(FLAVOUR), kyuubi)
 	make kyuubi
-else ifeq ($(FLAVOUR), spark-gpu)
-	make spark-gpu
+else ifeq ($(FLAVOUR), gpu)
+	make gpu
 else
 	make spark
 endif
@@ -305,7 +306,7 @@ tests: $(K8S_MARKER) $(AWS_MARKER) $(AZURE_MARKER)
 	@echo "=== Running Integration Tests ==="
 ifeq ($(FLAVOUR), jupyter)
 	/bin/bash ./tests/integration/integration-tests-jupyter.sh
-else ifeq ($(FLAVOUR), spark-gpu)
+else ifeq ($(FLAVOUR), gpu)
 	/bin/bash ./tests/integration/integration-tests-gpu.sh
 else ifeq ($(FLAVOUR), kyuubi)
 	@export AZURE_STORAGE_ACCOUNT=$(AZURE_STORAGE_ACCOUNT) \

images/metadata.yaml

@@ -11,4 +11,14 @@ flavours:
     version: 1.9.0
     image_description: |
       This is an OCI image that contains Kyuubi, fully integrated with Charmed
-      Spark ecosystem and utilities.
+      Spark ecosystem and utilities.
+  gpu:
+    version: 24.04.1
+    image_description: |
+      This is an OCI image that bundles Apache Spark binaries together with other
+      tools of its ecosystem in order to be used in Charmed Operators, providing
+      an automated and seamless experience to deploy, operate, manage and monitor
+      SparkJob on K8s cluster.
+      This image includes support for Nvidia Spark Rapids.
+      It is an open source, end-to-end, production ready data platform on top of
+      cloud native technologies.

tests/integration/integration-tests-gpu.sh

@@ -25,7 +25,11 @@ S3_BUCKET=spark-$(uuidgen)
 
 get_spark_version(){
   # Fetch Spark version from rockcraft.yaml
-  cat images/charmed-spark-gpu/rockcraft.yaml | yq '(.version)' 
+  SPARK_VERSION=$(cat images/charmed-spark/rockcraft.yaml | yq '(.version)')
+
+  GPU_VERSION=$(cat images/metadata.yaml | yq .flavours.gpu.version)
+
+  echo "${SPARK_VERSION}-${GPU_VERSION}"
 }
 
 
@@ -43,8 +47,10 @@ setup_user() {
 
   create_serviceaccount_using_pod $USERNAME $NAMESPACE $ADMIN_POD_NAME
 
+  IMAGE=$(spark_image)
+
   # Create the pod with the Spark service account
-  cat ./tests/integration/resources/testpod.yaml | yq ea '.spec.serviceAccountName = '\"${USERNAME}\"' | .spec.containers[0].image="ghcr.io/welpaolo/charmed-spark@sha256:d8273bd904bb5f74234bc0756d520115b5668e2ac4f2b65a677bfb1c27e882da"' | \
+  cat ./tests/integration/resources/testpod.yaml | yq ea '.spec.serviceAccountName = '\"${USERNAME}\"' | .spec.containers[0].image='\"${IMAGE}\" | \
     kubectl -n tests apply -f -
 
   wait_for_pod testpod $NAMESPACE