Merge pull request #77 from pipatth/master

Disable ory and prefect create-tenant for minikube

CHANGELOG.md

@@ -9,6 +9,9 @@
 - Use specific chart versions and image tags for MLFlow, Ambassador, and PostgresQL
 - Remove terraform_backend.tf to prevent applying from the OpenMLOPs repository
 
+### Enhancements
+- Add tutorial on how to set up on Minikube
+
 ## 0.1.0
 
 ### Features

main.tf

@@ -79,6 +79,7 @@ module "prefect-server" {
   service_account_name = var.prefect_service_account_name
   seldon_manager_cluster_role_name = var.install_seldon ? "seldon-manager-role-${var.seldon_namespace}" : ""
   feast_spark_operator_cluster_role_name = var.install_feast ? var.feast_spark_operator_cluster_role_name : ""
+  create_tenant_enabled = var.prefect_create_tenant_enabled
 }
 
 
@@ -172,6 +173,7 @@ resource "kubernetes_namespace" "ory_namespace" {
 }
 
 module "ory" {
+  count = var.enable_ory_authentication ? 1 : 0
   source = "./modules/ory"
   namespace = kubernetes_namespace.ory_namespace[0].metadata[0].name
   cookie_secret = var.ory_kratos_cookie_secret

modules/prefect-server/main.tf

@@ -9,7 +9,7 @@ resource "helm_release" "prefect-server" {
 
   set {
     name = "jobs.createTenant.enabled"
-    value = true
+    value = var.create_tenant_enabled
   }
 
   set {
@@ -89,7 +89,7 @@ resource "helm_release" "prefect-server" {
 
   set {
     name = "apollo.service.type"
-    value = var.service_type
+    value = "ClusterIP"
   }
 
   set {

modules/prefect-server/variables.tf

@@ -26,6 +26,11 @@ variable "annotations" {
   default     = {}
 }
 
+variable "create_tenant_enabled" {
+  description = "determines if the Prefect tenant is created automatically"
+  default     = true
+}
+
 variable "agent_enabled" {
   description = "determines if the Prefect Kubernetes agent is deployed"
   default     = true

tutorials/set-up-minikube-cluster.md

@@ -0,0 +1,144 @@
+# Set up your Minikube cluster in under 30 minutes
+
+## Step 1: Cloning the repositories
+
+On your local machine, create a directory called `openmlops` and clone the following three Open MLOps repositories into that directory.
+
+```
+git clone [email protected]:datarevenue-berlin/OpenMLOps.git
+```
+
+## Step 2: Starting a minikube cluster locally
+
+After you install minikube as described [here](https://minikube.sigs.k8s.io/docs/start/), you can start your minikube cluster by running:
+
+```
+minikube start --kubernetes-version=v1.17.17
+```
+
+Then, start a [minikube tunnel](https://minikube.sigs.k8s.io/docs/handbook/accessing/#using-minikube-tunnel) by running:
+
+```
+minikube tunnel
+```
+
+## Step 3: Create a `my_vars.tfvars` file
+
+Next you'll need to personalise the secrets and other values in the tfvars file. You can create  a file in `openmlops/OpenMLOps/my_vars.tfvars` and use the example settings below.
+
+
+```
+aws = false
+db_username = "mlflow-db-user"
+db_password = "mlflow-db-pasword"
+hostname = "myambassador.com"
+ory_kratos_cookie_secret = "secret"
+ory_kratos_db_password = "password"
+install_metrics_server = false
+install_feast = false
+install_seldon = false
+prefect_create_tenant_enabled = false
+jhub_proxy_secret_token = "IfYouDecideToUseJhubProxyYouShouldChangeThisValueToARandomString"
+enable_ory_authentication = false
+oauth2_providers = []
+mlflow_artifact_root = "/tmp"
+```
+
+## Step 4: Initialising Terraform
+
+Run the following in your shell to set your Kubectl config.
+
+```
+export KUBE_CONFIG_PATH=~./kube/config
+```
+
+Now change into the `OpenMLOps` directory and run `terraform init`, which will pull down the terraform dependencies that you need.
+
+You should see "Terraform has been successfully initialized!" towards the end of the output.
+
+### Initialising the Kubernetes secrets
+Before you continue with the next steps of Terraform, you need to generate some secrets for Kubernetes by using the following commands:
+
+```
+sh generate-secrets.sh
+```
+
+## Step 5: Creating the infrastructure
+
+Now you can run
+
+```
+terraform apply -var-file=my_vars.tfvars
+```
+
+Note that this step will actually create the resources on your minikube cluster.
+
+Enter `yes` when prompted and wait for everything to create. You can follow along as it makes progress, or go make coffee and wait for it to finish.
+
+## Step 6: Creating Prefect tenant
+
+Prefect tenant is not created automatically when running on minikube. To create a tenant, following the following steps:
+
+- Install prefect locally
+
+```
+pip install prefect
+```
+
+- In another terminal session, port forward the graphql
+
+```
+kubectl port-forward -n prefect svc/prefect-server-apollo 4200
+```
+
+- Run the command below to create a tenant. You should see a message that a tenant is created.
+
+```
+prefect backend server && prefect server create-tenant --name default --slug default
+```
+
+## Trying out the services
+
+The first step is to look at the service addresses and ports available. Run the command
+
+```
+kubectl get services --all-namespaces
+```
+
+You should look for the `EXTERNAL_IP` and `PORT` of the `LoadBalancer` services you want to access. For example,
+
+* `http://[EXTERNAL_IP_OF_JUPYTER_HUB_PROXY_PUBLIC]:80` to start writing code in your setup
+* `http://[EXTERNAL_IP_OF_PREFECT_UI]:8080` to configure workflow and dataflows
+* `http://[EXTERNAL_IP_OF_MLFLOW]:5000` to see and track your experiments
+
+Next, take a look at [our tutorial](./basic-usage-of-jupyter-mlflow-and-prefect.md) on creating a basic production machine learning system using the Open MLOps architecture.
+
+## Tearing down the cluster
+
+To destroy all the resources, run the following command:
+
+```
+terraform destroy -var-file=my_vars.tfvars
+```
+
+To delete the minikube cluster, run the following commands:
+
+```
+minikube stop
+minikube delete
+```
+
+## Troubleshooting
+
+### Terraform keeps waiting on resources to become ready and then fails
+
+This may be caused by External IPs not being assigned to the services that expect them.
+- Do you have `minikube tunnel` running? Isn't it asking for sudo password?
+- On Macs, when using Docker driver to run minikube, `minikube tunnel` doesn't behave very well. Try using e.g. Virtualbox. Please refer to `minikube` documentation.
+
+### Kubectl hangs, Terraform exits with `the server was unable to return a response in the time allotted, but may still be processing the request`
+
+You may have allocated too little resources for the minikube cluster. Try starting the cluster with:
+```commandline
+minikube start --kubernetes-version=v1.17.17 --memory 8192 --cpus 2 start
+```

variables.tf

@@ -72,6 +72,11 @@ variable "prefect_service_account_name" {
   default = "prefect-server-serviceaccount"
 }
 
+variable "prefect_create_tenant_enabled" {
+  description = "determines if the Prefect tenant is created automatically"
+  default     = true
+}
+
 ## Jupyter Hub
 
 variable "jupyter_dummy_password" {