diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 1007cc36..07e0fc04 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -52,5 +52,9 @@ jobs: # Import artifact into microk8s to be used in integration tests sudo make import TARGET=microk8s PREFIX=test- REPOSITORY=ghcr.io/canonical/ \ -o $(find .make_cache -name "*.tag") + + # Import artifact into docker with new tag + sudo make jupyter TARGET=docker REPOSITORY=ghcr.io/canonical/ PREFIX=test- \ + -o $(find .make_cache -name "*.tag") sg microk8s -c "make tests" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 8ada9fd3..4868dcec 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -86,22 +86,50 @@ jobs: RISK=${{ needs.release_checks.outputs.risk }} TRACK=${{ needs.release_checks.outputs.track }} if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi - + + IMAGE_NAME=$(make REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Image\:" | cut -d ":" -f2 | xargs) + # Import artifact into docker with new tag sudo make import TARGET=docker REPOSITORY=${REPOSITORY} TAG=${TAG}\ -o ${{ steps.artifact.outputs.name }} - IMAGE_NAME="${REPOSITORY}charmed-spark" - echo "Publishing ${IMAGE_NAME}:${TAG}" docker push ${IMAGE_NAME}:${TAG} if [[ "$RISK" == "edge" ]]; then - TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge" + VERSION_TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge" - sudo make import TARGET=docker REPOSITORY=${REPOSITORY} TAG=${TAG}\ - -o ${{ steps.artifact.outputs.name }} + docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} + + echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" + docker push ${IMAGE_NAME}:${VERSION_TAG} + fi + + - name: Publish JupyterLab Image to Channel + run: | + + REPOSITORY="ghcr.io/canonical/" + RISK=${{ needs.release_checks.outputs.risk }} + TRACK=${{ needs.release_checks.outputs.track }} + if [ ! -z "$RISK" ] && [ "${RISK}" != "no-risk" ]; then TAG=${TRACK}_${RISK}; else TAG=${TRACK}; fi + + # Import artifact into docker with new tag + sudo make jupyter REPOSITORY=${REPOSITORY} TAG=${TAG}\ + -o $(find .make_cache -name "*.tag") + + IMAGE_NAME=$(make REPOSITORY=${REPOSITORY} TAG=${TAG} help | grep "Jupyter\:" | cut -d ":" -f2 | xargs) + + echo "Publishing ${IMAGE_NAME}:${TAG}" + docker push ${IMAGE_NAME}:${TAG} + + if [[ "$RISK" == "edge" ]]; then + VERSION_TAG="${{ needs.release_checks.outputs.version }}-${{ needs.release_checks.outputs.base }}_edge" + + docker tag ${IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${VERSION_TAG} - echo "Publishing ${IMAGE_NAME}:${TAG}" - docker push ${IMAGE_NAME}:${TAG} - fi \ No newline at end of file + echo "Publishing ${IMAGE_NAME}:${VERSION_TAG}" + docker push ${IMAGE_NAME}:${VERSION_TAG} + fi + + + diff --git a/Makefile b/Makefile index 14a7fa9c..a1348b27 100644 --- a/Makefile +++ b/Makefile @@ -40,14 +40,20 @@ _TMP_OCI_TAG := $(_MAKE_DIR)/$(_TMP_OCI_NAME)/$(TAG).tag CHARMED_OCI_FULL_NAME=$(REPOSITORY)$(PREFIX)$(IMAGE_NAME) CHARMED_OCI_TAG := $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tag +CHARMED_OCI_JUPYTER=$(CHARMED_OCI_FULL_NAME)-jupyterlab4 +CHARMED_OCI_JUPYTER_TAG := $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tag + help: @echo "---------------HELP-----------------" - @echo "Image: $(IMAGE_NAME)" + @echo "Name: $(IMAGE_NAME)" @echo "Version: $(VERSION)" @echo "Platform: $(PLATFORM)" @echo " " @echo "Artifact: $(BASE_NAME)" @echo " " + @echo "Image: $(CHARMED_OCI_FULL_NAME)" + @echo "Jupyter: $(CHARMED_OCI_JUPYTER)" + @echo " " @echo "Type 'make' followed by one of these keywords:" @echo " " @echo " - build for creating the OCI Images" @@ -69,8 +75,8 @@ $(_TMP_OCI_TAG): $(_ROCK_OCI) if [ ! -d "$(_MAKE_DIR)/$(_TMP_OCI_NAME)" ]; then mkdir -p "$(_MAKE_DIR)/$(_TMP_OCI_NAME)"; fi touch $(_TMP_OCI_TAG) -$(CHARMED_OCI_TAG): $(_TMP_OCI_TAG) - docker build - -t "$(CHARMED_OCI_FULL_NAME):$(TAG)" --build-arg BASE_IMAGE="$(_TMP_OCI_NAME):$(TAG)" < Dockerfile +$(CHARMED_OCI_TAG): $(_TMP_OCI_TAG) build/Dockerfile + docker build -t "$(CHARMED_OCI_FULL_NAME):$(TAG)" --build-arg BASE_IMAGE="$(_TMP_OCI_NAME):$(TAG)" -f build/Dockerfile . if [ ! -d "$(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)" ]; then mkdir -p "$(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)"; fi touch $(CHARMED_OCI_TAG) @@ -85,12 +91,21 @@ microk8s: $(K8S_TAG) $(_MAKE_DIR)/%/$(TAG).tar: $(_MAKE_DIR)/%/$(TAG).tag docker save $*:$(TAG) > $(_MAKE_DIR)/$*/$(TAG).tar +$(CHARMED_OCI_JUPYTER_TAG): $(CHARMED_OCI_TAG) build/Dockerfile.jupyter + docker build -t "$(CHARMED_OCI_JUPYTER):$(TAG)" --build-arg BASE_IMAGE="$(CHARMED_OCI_FULL_NAME):$(TAG)" -f build/Dockerfile.jupyter . + if [ ! -d "$(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)" ]; then mkdir -p "$(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)"; fi + touch $(CHARMED_OCI_JUPYTER_TAG) + $(BASE_NAME): $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tar @echo "=== Creating $(BASE_NAME) OCI archive ===" cp $(_MAKE_DIR)/$(CHARMED_OCI_FULL_NAME)/$(TAG).tar $(BASE_NAME) build: $(BASE_NAME) +jupyter: $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tar + @echo "=== Creating $(BASE_NAME) OCI jupyter archive ===" + cp $(_MAKE_DIR)/$(CHARMED_OCI_JUPYTER)/$(TAG).tar $(IMAGE_NAME)-jupyter_$(VERSION)_$(PLATFORM).tar + ifeq ($(TARGET), docker) import: build @echo "=== Importing image $(CHARMED_OCI_FULL_NAME):$(TAG) into docker ===" diff --git a/README.md b/README.md index 6c68b3d6..922426f1 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,29 @@ Charmed Spark Rock Image is delivered with Pebble already included in order to m docker run ghcr.io/canonical/charmed-spark:3.4.2-22.04_edge \; start history-server ``` +### Running Jupyter Lab + +In the Charmed Spark bundle we also provide the `charmed-spark-jupyter` image, +specifically built for running JupterLab server integrated with Spark where any notebook will also +start dedicated executors and inject a SparkSession and/or SparkContext within the notebook. + +To start a JupyterLab server using the `charmed-spark-jupyter` image, use + +```shell +docker run \ + -v /path/to/kube/config:/var/lib/spark/.kube/config \ + -p :8888 + ghcr.io/canonical/charmed-spark-jupyter:3.4.1-22.04_edge \ + --username --namespace +``` + +Make sure to have created the `` in the `` with the `spark8t` CLI beforehand. +You should be able to access the jupyter server at `http://0.0.0.0:`. + +You can provide extra-arguments to further configure the spark-executors by providing more `spark8t` +commands. The mount of the local `kubeconfig` file is necessary to provide the ability to the +JupyterLab server to act as a Spark driver and request resources on the K8s cluster. + ## Developers and Contributing Please see the [CONTRIBUTING.md](https://github.com/canonical/charmed-spark-rock/blob/3.4-22.04/edge/CONTRIBUTING.md) for guidelines and for developer guidance. diff --git a/Dockerfile b/build/Dockerfile similarity index 100% rename from Dockerfile rename to build/Dockerfile diff --git a/build/Dockerfile.jupyter b/build/Dockerfile.jupyter new file mode 100644 index 00000000..b87d49e0 --- /dev/null +++ b/build/Dockerfile.jupyter @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=base-charmed-spark:latest +FROM $BASE_IMAGE + +USER root +RUN rm /var/lib/pebble/default/layers/*.yaml + +RUN python3 -m pip install "jupyterlab~=4.0" +COPY ./files/jupyter/pebble/layers.yaml /var/lib/pebble/default/layers/001-charmed-jupyter.yaml + +USER _daemon_ + +# Provide Default Entrypoint for Pebble +ENTRYPOINT [ "/bin/pebble", "enter", "--verbose", "--args", "jupyter" ] \ No newline at end of file diff --git a/files/jupyter/pebble/layers.yaml b/files/jupyter/pebble/layers.yaml new file mode 100644 index 00000000..d2fe4ba4 --- /dev/null +++ b/files/jupyter/pebble/layers.yaml @@ -0,0 +1,9 @@ +services: + jupyter: + command: "spark-client.pyspark" + summary: "This is the Spark-powered Jupyter service" + override: replace + startup: enabled + environment: + PYSPARK_DRIVER_PYTHON: jupyter + PYSPARK_DRIVER_PYTHON_OPTS: "lab --no-browser --port=8888 --ip=0.0.0.0 --NotebookApp.token='' --notebook-dir=/var/lib/spark/notebook"