Merge pull request #55 from datarevenue-berlin/MLOPS-291-fix-dask-jup…

…yter

Daskhub deployment, deprecating Jupyter + Dask

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 - Support password recovery using SMTP
+- Support Dask from Jupyter notebook
 
 ### Bug fixes
 - Use specific chart versions and image tags for MLFlow, Ambassador, and PostgresQL

docker/openmlops-notebook/Dockerfile

@@ -0,0 +1,8 @@
+FROM jupyter/scipy-notebook:hub-1.4.2
+RUN set -ex \
+   && pip install \
+   'prefect==0.14.12' \
+   'mlflow==1.14.1' \
+   'seldon-core==1.6.0' \
+   'dask==2021.7.0' \
+   'dask-kubernetes==2021.3.1'

main.tf

@@ -1,34 +1,68 @@
-resource "kubernetes_namespace" "jupyterhub_namespace" {
+resource "kubernetes_namespace" "daskhub_namespace" {
   metadata {
-    name = var.jupyterhub_namespace
+    name = "daskhub"
   }
 }
 
-module "jupyterhub" {
-  count     = var.install_jupyterhub ? 1 : 0
-  source    = "./modules/jupyterhub"
-  namespace = kubernetes_namespace.jupyterhub_namespace.metadata[0].name
+module "dask-jupyterhub" {
+    source    = "./modules/dask-jupyterhub"
+    namespace = kubernetes_namespace.daskhub_namespace.metadata[0].name
+}
 
-  # Proxy settings
-  proxy_secret_token                    = var.jhub_proxy_secret_token
-  proxy_https_enabled                   = var.jhub_proxy_https_enabled
-  proxy_https_hosts                     = var.jhub_proxy_https_hosts
-  proxy_https_letsencrypt_contact_email = var.jhub_proxy_https_letsencrypt_contact_email
-  proxy_service_type                    = var.jhub_proxy_service_type
+resource "kubernetes_service_account" "daskhub-sa" {
+  metadata {
+    name      = "daskhub-sa"
+    namespace = kubernetes_namespace.daskhub_namespace.metadata[0].name
+  }
+}
 
-  # Authentication settings
-  # Following values should be `null` if oauth_github is disabled. However we need to pass submodule's defaults here
-  # explicitly because of this Terraform bug: https://github.com/hashicorp/terraform/issues/21702
-  authentication_type   = var.oauth_github_enable ? "github" : "dummy"
-  authentication_config = merge(
-    local.jhub_auth_config,
-    {JupyterHub = {authenticator_class = var.oauth_github_enable ? "github" : "dummy"}}
-  )
+resource "kubernetes_role" "daskhub-role" {
+  metadata {
+    name = "daskhub-role"
+    namespace = kubernetes_namespace.daskhub_namespace.metadata[0].name
+  }
 
-  # Profile list configuration
-  singleuser_profile_list = var.singleuser_profile_list
+  rule {
+    api_groups     = [""]
+    resources      = ["pods"]
+    verbs          = ["get", "list", "watch", "create", "delete"]
+  }
+
+  rule {
+    api_groups     = [""]
+    resources      = ["pods/logs"]
+    verbs          = ["get", "list"]
+  }
+
+  rule {
+    api_groups     = [""]
+    resources      = ["services"]
+    verbs          = ["get", "list", "watch", "create", "delete"]
+  }
+
+  rule {
+    api_groups     = ["policy"]
+    resources      = ["poddisruptionbudgets"]
+    verbs          = ["get", "list", "watch", "create", "delete"]
+  }
 }
 
+resource "kubernetes_role_binding" "daskhub-rb" {
+  metadata {
+    name = "daskhub-rb"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = "daskhub-role"
+  }
+
+  subject {
+    kind = "ServiceAccount"
+    name = "daskhub-sa"
+  }
+}
 
 resource "kubernetes_namespace" "mlflow_namespace" {
   metadata {
@@ -90,7 +124,6 @@ resource "kubernetes_namespace" "dask_namespace" {
 }
 
 module "dask" {
-
   source    = "./modules/dask"
   namespace = kubernetes_namespace.dask_namespace.metadata[0].name
 
@@ -102,7 +135,7 @@ module "dask" {
   worker_environment_variables = [
     {
       name  = "EXTRA_PIP_PACKAGES"
-      value = "prefect==0.14.1 --upgrade"
+      value = "prefect==0.14.1 aiohttp --upgrade"
     }
   ]
 }

modules/ambassador/values.yaml

@@ -57,7 +57,7 @@ service:
       apiVersion: getambassador.io/v2
       kind: Mapping
       name: jhub_mapping
-      service: proxy-public.jhub:80
+      service: proxy-public.daskhub:80
       prefix: /
       host: jupyter.${hostname}
       allow_upgrade:

modules/dask-jupyterhub/main.tf

@@ -0,0 +1,23 @@
+resource "helm_release" "dask-jupyterhub" {
+  name      = "daskhub"
+  namespace = var.namespace
+
+  repository = "https://helm.dask.org"
+  chart      = "daskhub"
+  version    = "2021.7.2"
+
+  // It takes some time to pull all the necessary images.
+  timeout = 15 * 60
+
+  values = [templatefile("${path.module}/values.yaml", {
+    jupyterhub_secret = var.jupyterhub_secret
+    daskgateway_secret = var.daskgateway_secret
+    singleuser_profile_list = var.singleuser_profile_list
+    singleuser_image_pull_secrets = var.singleuser_image_pull_secrets
+    singleuser_image_pull_policy = var.singleuser_image_pull_policy
+    singleuser_memory_guarantee = var.singleuser_memory_guarantee
+    singleuser_storage_capacity = var.singleuser_storage_capacity
+    singleuser_storage_mount_path = var.singleuser_storage_mount_path
+    hub_allow_named_servers = var.hub_allow_named_servers
+  })]
+}

modules/dask-jupyterhub/values.yaml

@@ -0,0 +1,32 @@
+jupyterhub:
+  proxy:
+    service:
+      type: "ClusterIP"
+  hub:
+    allowNamedServers: ${ hub_allow_named_servers }
+  singleuser:
+    serviceAccountName: "daskhub-sa"
+    profileList:
+%{for profile in singleuser_profile_list ~}
+    - display_name: ${ profile.display_name }
+      description: "${ profile.description }"
+      default: ${ profile.default }
+      kubespawner_override:
+        image: ${ profile.kubespawner_override.image }
+%{ endfor ~}
+    memory:
+      guarantee: ${ singleuser_memory_guarantee }
+    storage:
+      capacity: ${ singleuser_storage_capacity }
+      homeMountPath: ${ singleuser_storage_mount_path }
+    extraEnv:
+      TZ: "Europe/Berlin"
+
+dask-gateway:
+  enabled: false
+  gateway:
+    auth:
+      type: null
+
+dask-kubernetes:
+  enabled: true

modules/dask-jupyterhub/variables.tf

@@ -0,0 +1,71 @@
+variable "namespace" {
+  description = "Namespace name to deploy the application"
+  default     = "default"
+}
+
+variable "jupyterhub_secret" {
+    type = string
+    default = "4301bf5a2aa1fbade157046863ac64ec46df03e3da39ec3bf345a2f8caa81e02"
+}
+
+variable "daskgateway_secret" {
+    type = string
+    default = "ca7de235a4ae54103d49f5004a11690004c66fe14810f35dc476103573e56ff1"
+}
+
+variable "singleuser_image_pull_secrets" {
+  type = list(
+    object({
+      name = string
+    })
+  )
+  default = []
+}
+
+variable "singleuser_image_pull_policy" {
+  default = "Always"
+}
+
+variable "singleuser_default_url" {
+  description = ""
+  default     = "/lab"
+}
+
+variable "singleuser_profile_list" {
+  description = "List of images which the user can select to spawn a server"
+  type = list(
+    object({
+      display_name = string
+      description  = string
+      default      = bool
+      kubespawner_override = object({
+        image = string
+      })
+  }))
+
+  default = [{
+      display_name = "OpenMLOps client environment"
+      description  = "Notebook with OpenMLOps required client libraries installed. <br> Image: drtools/openmlops-notebook:v1.4"
+      default      = true
+      kubespawner_override = {
+        image = "drtools/openmlops-notebook:v1.4"
+      }
+  }]
+}
+
+variable "singleuser_memory_guarantee" {
+  default = "1G"
+}
+
+variable "singleuser_storage_capacity" {
+  default = "1G"
+}
+
+variable "singleuser_storage_mount_path" {
+  default = "/home/jovyan/persistent"
+}
+
+variable "hub_allow_named_servers" {
+  description = "Configures if a user can spawn multiple servers"
+  default     = false
+}

modules/dask-jupyterhub/worker.yaml

@@ -0,0 +1,18 @@
+kind: Pod
+spec:
+  restartPolicy: Never
+  containers:
+  - image: daskdev/dask:2021.3.1
+    imagePullPolicy: IfNotPresent
+    args: [dask-worker, --nthreads, '2', --no-dashboard, --memory-limit, 6GB, --death-timeout, '60']
+    name: dask
+    env:
+      - name: EXTRA_PIP_PACKAGES
+        value: git+https://github.com/dask/distributed
+    resources:
+      limits:
+        cpu: "0.5"
+        memory: 1G
+      requests:
+        cpu: "0.5"
+        memory: 1G

modules/dask/main.tf

@@ -7,67 +7,17 @@ resource "helm_release" "dask" {
   chart      = "dask"
   version    = "2021.7.0"
 
-  set {
-    name  = "worker.name"
-    value = var.worker_name
-  }
-
-  set {
-    name  = "worker.replicas"
-    value = var.worker_replicas
-  }
-
-  set {
-    name  = "worker.image.repository"
-    value = var.worker_image_repository
-  }
-
-  set {
-    name  = "worker.image.tag"
-    value = var.worker_image_tag
-  }
-
-  set {
-    name  = "worker.image.pullPolicy"
-    value = var.worker_image_pull_policy
-  }
-
-  set {
-    name  = "worker.image.dask_worker"
-    value = var.worker_image_dask_worker_command
-  }
-
-  set {
-    name  = "scheduler.image.repository"
-    value = var.scheduler_image_repository
-  }
-
-  set {
-    name  = "scheduler.image.tag"
-    value = var.scheduler_image_tag
-  }
-
-  set {
-    name  = "scheduler.image.pullPolicy"
-    value = var.scheduler_image_pull_policy
-  }
-
-
-  values = [
-    yamlencode({
-      "worker" = {
-        "env" = var.worker_environment_variables
-      }
-    }),
-    yamlencode({
-      "worker" = {
-        "image" = {
-          "pullSecrets" = var.worker_image_pull_secret
-        }
-      }
-    })
-  ]
-
-
-
+  values = [templatefile("${path.module}/values.yaml", {
+    worker_name = var.worker_name
+    worker_replicas = var.worker_replicas
+    worker_environment_variables = var.worker_environment_variables
+    worker_image_repository = var.worker_image_repository
+    worker_image_tag = var.worker_image_tag
+    worker_image_pull_policy = var.worker_image_pull_policy
+    worker_image_pull_secret = var.worker_image_pull_secret
+    worker_image_dask_worker_command = var.worker_image_dask_worker_command
+    scheduler_image_repository = var.scheduler_image_repository
+    scheduler_image_tag = var.scheduler_image_tag
+    scheduler_image_pull_policy = var.scheduler_image_pull_policy
+  })]
 }