diff --git a/.gitignore b/.gitignore
index 7c437c2..ee3f2dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,5 @@
__pycache__
*.sw[op]
*.py[oc]
+.env
+/venv
diff --git a/.idea/.gitignore b/.idea/.gitignore
new file mode 100644
index 0000000..26d3352
--- /dev/null
+++ b/.idea/.gitignore
@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml
diff --git a/.idea/celery-kubernetes-example.iml b/.idea/celery-kubernetes-example.iml
new file mode 100644
index 0000000..4fe1e5d
--- /dev/null
+++ b/.idea/celery-kubernetes-example.iml
@@ -0,0 +1,14 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml
new file mode 100644
index 0000000..b90389d
--- /dev/null
+++ b/.idea/inspectionProfiles/Project_Default.xml
@@ -0,0 +1,51 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml
new file mode 100644
index 0000000..105ce2d
--- /dev/null
+++ b/.idea/inspectionProfiles/profiles_settings.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 0000000..140ea33
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 0000000..18a2765
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 0000000..94a25f7
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/GKE_README.txt b/GKE_README.txt
new file mode 100644
index 0000000..894207b
--- /dev/null
+++ b/GKE_README.txt
@@ -0,0 +1,63 @@
+make sure gcloud sdk is installed and pathed correctly
+
+gcloud auth login
+gcloud projects list
+gcloud config set project MY_PROJECT_NAME
+
+
+# push the images to google contianer registry
+gcloud auth configure-docker
+
+docker build --tag myproject:1 --file myproject/Dockerfile .
+docker tag myproject:1 gcr.io/first-304221/myproject:1
+docker push gcr.io/first-304221/myproject:1
+
+docker build --tag consumer-small:1 --file consumer-small/Dockerfile .
+docker tag consumer-small:1 gcr.io/first-304221/consumer-small:1
+docker push gcr.io/first-304221/consumer-small:1
+
+docker build --tag consumer-large:1 --file consumer-large/Dockerfile .
+docker tag consumer-large:1 gcr.io/first-304221/consumer-large:1
+docker push gcr.io/first-304221/consumer-large:1
+
+
+# GKE
+gcloud components install kubectl
+
+gcloud config set project MY_PROJECT_NAME
+gcloud config set compute/zone ZONE_NAME
+
+
+# -------- my-celery-cluster -------------
+gcloud container clusters create my-celery-cluster --num-nodes=1 \
+--preemptible \
+--enable-autoscaling --max-nodes=5 --min-nodes=1
+
+gcloud container clusters get-credentials my-celery-cluster
+
+
+kubectl create -f message_queue/rabbitmq-deployment.yaml
+kubectl create -f message_queue/rabbitmq-service.yaml
+
+
+# I changed the yml to point to the image in gcr
+kubectl create -f myproject/deployment.yaml
+kubectl create -f consumer-large/deployment.yaml
+
+kubectl get pods
+
+
+# to find external ip
+kubectl get service myproject-svc
+
+
+# DELETE CLUSTER to avoid addtional charges
+gcloud container clusters delete my-celery-cluster
+
+
+
+
+
+
+
+
diff --git a/README.md b/README.md
index a2d9ecd..3458719 100644
--- a/README.md
+++ b/README.md
@@ -36,23 +36,61 @@ Assuming `dockerd` is running and `minikube` is installed, let's deploy the syst
### Initialization
-Create a Minikube cluster that uses the local `dockerd` environment:
+Make sure you have google cloud sdk installed and pathed correctly.
+https://cloud.google.com/sdk/docs/install
+
+then auth you google credentials and either make a new project or use an exiting one.
+```
+gcloud auth login
+gcloud projects list
+gcloud projects create example-foo-bar-project-name-1 --name="Happy project"
+```
+or use an exiting project
+```
+gcloud config set project MY_PROJECT_NAME
+```
+
+run this to allow you to push to google container registry and when prompted say yes.
```
-minikube start
-eval $(minikube -p minikube docker-env)
+gcloud auth configure-docker
```
-Build all Docker images for Minikube:
+
+
+Build and push you Docker images to GRC:
```
docker build --tag myproject:1 --file myproject/Dockerfile .
+docker tag myproject:1 gcr.io/first-304221/myproject:1
+docker push gcr.io/first-304221/myproject:1
+
docker build --tag consumer-small:1 --file consumer-small/Dockerfile .
+docker tag consumer-small:1 gcr.io/first-304221/consumer-small:1
+docker push gcr.io/first-304221/consumer-small:1
+
docker build --tag consumer-large:1 --file consumer-large/Dockerfile .
+docker tag consumer-large:1 gcr.io/first-304221/consumer-large:1
+docker push gcr.io/first-304221/consumer-large:1
```
-Check that the images were created successfully:
+
+
+### Deploying applications
+
+Install gcloud kubectl and set project
+```
+gcloud components install kubectl
+
+gcloud config set project MY_PROJECT_NAME
+gcloud config set compute/zone ZONE_NAME
```
-docker images
+
+Create a GKE cluster with preemptive (spot) instance processors. And pass credentials to kubectl.
+```
+gcloud container clusters create my-celery-cluster --num-nodes=1 \
+--preemptible \
+--enable-autoscaling --max-nodes=5 --min-nodes=1
+
+gcloud container clusters get-credentials my-celery-cluster
```
-### Deploying applications
Deploy the RabbitMQ message broker as a service inside the cluster (I left some config files for Redis too but they are not used here):
```
@@ -106,11 +144,11 @@ I prefer to open new terminals or `tmux` for all applications and then use `kube
### Interacting with the web app
-Now everything is running and we can expose the Flask web app port to our local machine:
+Now everything is running and we can get the exposed port from the Load balancer service
```
-kubectl port-forward deployment/myproject 5000:5000
+kubectl get service myproject-svc
```
-Then open http://localhost:5000/ in a browser and you should see a simple web UI.
+Then got to the exteranl ip in a browser, and you should see a simple web UI.
Try copy-pasting some strings and compute the longest common substrings for them.
E.g. first try short strings and check that the tasks show up in the Celery logs of pod `consumer-small`.
@@ -155,9 +193,8 @@ celery inspect report --broker=$CELERY_BROKER_URL
```
## Cleanup
-Terminate all pods by removing the deployments:
+Terminate the cluster to avoid any additional charges
```
-kubectl delete deploy myproject consumer-large rabbitmq
-kubectl delete service rabbitmq-service
+gcloud container clusters delete my-celery-cluster
```
diff --git a/consumer-large/celery_app.py b/consumer-large/celery_app.py
index a4a5b19..b84f3ef 100644
--- a/consumer-large/celery_app.py
+++ b/consumer-large/celery_app.py
@@ -5,6 +5,7 @@
from celery import Celery
from lcs.tasks import longest_common_substr
+
def make_celery():
return Celery("consumer-large",
backend=os.environ.get("CELERY_RESULT_BACKEND"),
diff --git a/consumer-large/deployment.yaml b/consumer-large/deployment.yaml
index d777711..a069384 100644
--- a/consumer-large/deployment.yaml
+++ b/consumer-large/deployment.yaml
@@ -16,8 +16,9 @@ spec:
spec:
containers:
- name: consumer-large
- image: consumer-large:1
- imagePullPolicy: Never
+ # image: consumer-large:1
+ image: gcr.io/first-304221/consumer-large:1
+ imagePullPolicy: Always
env:
- name: CELERY_BROKER_URL
value: "amqp://guest:guest@rabbitmq-service:5672"
diff --git a/lcs/build/lib/lcs/__init__.py b/lcs/build/lib/lcs/__init__.py
new file mode 100644
index 0000000..24c1183
--- /dev/null
+++ b/lcs/build/lib/lcs/__init__.py
@@ -0,0 +1,12 @@
+def longest_common_substr(str_a, str_b):
+ longest_str = ''
+ for i in range(len(str_a)):
+ for j in range(len(str_b)):
+ current_str = ''
+ k = 0
+ while i + k < len(str_a) and j + k < len(str_b) and str_a[i + k] == str_b[j + k]:
+ current_str += str_a[i + k]
+ k += 1
+ if k > len(longest_str):
+ longest_str = current_str
+ return longest_str
diff --git a/lcs/build/lib/lcs/tasks.py b/lcs/build/lib/lcs/tasks.py
new file mode 100644
index 0000000..a03c969
--- /dev/null
+++ b/lcs/build/lib/lcs/tasks.py
@@ -0,0 +1,9 @@
+"""
+Celery task wrapper for longest_common_substr.
+"""
+import lcs
+from celery import shared_task
+
+@shared_task
+def longest_common_substr(str_a, str_b):
+ return lcs.longest_common_substr(str_a, str_b)
diff --git a/myproject/Dockerfile b/myproject/Dockerfile
index 1e689a9..61199c1 100644
--- a/myproject/Dockerfile
+++ b/myproject/Dockerfile
@@ -7,5 +7,6 @@ COPY myproject/templates /usr/src/myproject/templates
RUN apk add --no-cache python3 py3-pip bash \
&& pip install /usr/src/lcs \
&& pip install -r /usr/src/myproject/requirements.txt
+EXPOSE 5000
WORKDIR /usr/src/myproject
ENTRYPOINT python3 main.py
diff --git a/myproject/deployment.yaml b/myproject/deployment.yaml
index d841568..f0f5318 100644
--- a/myproject/deployment.yaml
+++ b/myproject/deployment.yaml
@@ -1,3 +1,19 @@
+# LoadBalancer Service
+# Enables the pods in a deployment to be accessible from outside the cluster
+apiVersion: v1
+kind: Service
+metadata:
+ name: myproject-svc
+spec:
+ selector:
+ app: myproject
+ ports:
+ - protocol: "TCP"
+ port: 80
+ targetPort: 5000
+ type: LoadBalancer
+
+---
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -21,8 +37,12 @@ spec:
emptyDir: {}
containers:
- name: myproject
- image: myproject:1
- imagePullPolicy: Never
+ # I added this
+ ports:
+ - containerPort: 5000
+ image: gcr.io/first-304221/myproject:1
+ # image: myproject:1
+ imagePullPolicy: Always
volumeMounts:
- mountPath: /data
name: tmp-database
@@ -39,8 +59,9 @@ spec:
# - name: CELERY_RESULT_BACKEND
# value: 'redis://redis-service:6379/1'
- name: consumer-small
- image: consumer-small:1
- imagePullPolicy: Never
+ # image: consumer-small:1
+ image: gcr.io/first-304221/consumer-small:1
+ imagePullPolicy: Always
volumeMounts:
- mountPath: /data
name: tmp-database
diff --git a/myproject/main.py b/myproject/main.py
index 2a6d026..504b7a0 100644
--- a/myproject/main.py
+++ b/myproject/main.py
@@ -7,11 +7,13 @@
import flask
+
import database
import settings
import tasks
+
flask_app = flask.Flask(__name__)
flask_app.logger.setLevel(logging.INFO)
flask_app.config.update(
@@ -24,6 +26,8 @@
def timestamp2iso(t):
return '' if t is None else datetime.datetime.fromtimestamp(t).isoformat()
+
+
@flask_app.route("/", methods=["GET"])
def get_index():
return flask.render_template("index.html")
@@ -58,4 +62,4 @@ def post_task():
flask_app.logger.info("Created database %s", settings.database_path)
else:
flask_app.logger.info("Using existing database %s", settings.database_path)
- flask_app.run()
+ flask_app.run(host='0.0.0.0', port='5000')
diff --git a/myproject/requirements.txt b/myproject/requirements.txt
index 1da6814..93c8c8d 100644
--- a/myproject/requirements.txt
+++ b/myproject/requirements.txt
@@ -1,3 +1,3 @@
Flask ~= 1.1.2
celery ~= 4.4.0
-redis ~= 3.5.3
+redis ~= 3.5.3
\ No newline at end of file
diff --git a/myproject/tasks.py b/myproject/tasks.py
index 632874c..6b871cf 100644
--- a/myproject/tasks.py
+++ b/myproject/tasks.py
@@ -4,6 +4,7 @@
import celery
import lcs.tasks
+
import database
import settings
@@ -23,6 +24,7 @@ def __call__(self, *args, **kwargs):
c.Task = ContextTask
return c
+
def create_lcs_task(str_a, str_b):
"""
Create new async task to compute lcs on str_a and str_b.