Merge pull request #52 from semi-technologies/horizontal-scaling

Horizontal scaling

.travis.yml

@@ -2,8 +2,9 @@ language: minimal
 
 before_install:
 # Install Helm
-- curl -s "https://raw.githubusercontent.com/helm/helm/v2.16.9/scripts/get" | bash -s -- --version 'v2.13.1'
-- helm init --stable-repo-url=https://charts.helm.sh/stable --client-only 
+  - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+  - chmod 700 get_helm.sh
+  - ./get_helm.sh
 
 install: true
 

weaviate/Chart.yaml

@@ -1,4 +1,4 @@
-apiVersion: v1
+apiVersion: v2
 name: weaviate
 description: A Helm Chart to run Weaviate with all dependencies
 home: https://github.com/semi-technologies/weaviate
@@ -7,10 +7,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 13.4.0
+version: 14.0.0-rc.0
 
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: v1.7.0
+appVersion: v1.8.0-rc.0

weaviate/templates/weaviateHeadlessService.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.service.name }}-headless
+spec:
+  type: ClusterIP
+  clusterIP: None
+  selector:
+    app: weaviate
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 7000

weaviate/templates/weaviateService.yaml

@@ -10,3 +10,11 @@ spec:
     - protocol: TCP
       port: 80
       targetPort: 8080
+{{ if eq .Values.service.type "LoadBalancer" -}}
+  {{- if gt (len .Values.service.loadBalancerSourceRanges) 0 }}
+  loadBalancerSourceRanges:
+    {{- range $_, $sourceRange := .Values.service.loadBalancerSourceRanges }}
+    - {{ $sourceRange }}
+    {{ end -}}
+  {{- end -}}
+{{- end }}

weaviate/templates/weaviateStatefulset.yaml

@@ -65,6 +65,13 @@ spec:
           - name: NER_INFERENCE_API
             value: {{ index .Values "modules" "ner-transformers" "inferenceUrl" }}
           {{ end }}
+          - name: CLUSTER_GOSSIP_BIND_PORT
+            value: "7000"
+          - name: CLUSTER_DATA_BIND_PORT
+            value: "7001"
+          - name: CLUSTER_JOIN
+            value: {{ .Values.service.name }}-headless.{{ .Release.Namespace }}.svc.cluster.local
+
         ports:
           - containerPort: 8080
         args:
@@ -79,7 +86,7 @@ spec:
         volumeMounts:
           - name: weaviate-config
             mountPath: /weaviate-config
-          - name: weaviate-persistence-data-vol
+          - name: weaviate-data
             mountPath: /var/lib/weaviate
         livenessProbe:
           httpGet:
@@ -97,10 +104,19 @@ spec:
         - name: weaviate-config
           configMap:
             {{ if .Values.custom_config_map.enabled }}name: {{ .Values.custom_config_map.name }} {{ else }}name: weaviate-config{{ end }}
-        - name: weaviate-persistence-data-vol
-          persistentVolumeClaim:
-            {{ if empty .Values.storage.fullnameOverride }}
-            claimName: weaviate-persistence-data
-            {{ else }}
-            claimName: {{ .Values.storage.fullnameOverride }}
-            {{ end }}
+        # - name: weaviate-persistence-data-vol
+        #   persistentVolumeClaim:
+        #     {{ if empty .Values.storage.fullnameOverride }}
+        #     claimName: weaviate-persistence-data
+        #     {{ else }}
+        #     claimName: {{ .Values.storage.fullnameOverride }}
+        #     {{ end }}
+  volumeClaimTemplates:
+  - metadata:
+      name: weaviate-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      storageClassName: {{ .Values.storage.storageClassName }}
+      resources:
+        requests:
+          storage: {{ .Values.storage.size }}

weaviate/values.yaml

@@ -7,11 +7,12 @@ image:
   # of weaviate. In accordance with Infra-as-code, you should pin this value
   # down and only change it if you explicitly want to upgrade the Weaviate
   # version.
-  tag: 1.7.0
+  tag: 1.8.0-rc.0
   repo: semitechnologies/weaviate
 
-# manually scale horizontal replicas of weaviate.
-# Also note: https://github.com/semi-technologies/weaviate/issues/725
+# Scale replicas of Weaviate. Note that as of v1.8.0 dynamic scaling is limited
+# to cases where no data is imported yet. Scaling down after importing data may
+# break usability. Full dynamic scaliblity will be added in a future release.
 replicas: 1
 resources:
   requests:
@@ -26,13 +27,15 @@ resources:
 # created, instead the one defined in fullnameOverride will be used
 storage:
   size: 32Gi
+  storageClassName: premium-rwo
 
 # The service controls how weaviate is exposed to the outside world. If you
 # don't want a public load balancer, you can also choose 'ClusterIP' to make
 # weaviate only accessible within your cluster.
 service:
   name: weaviate
   type: LoadBalancer
+  loadBalancerSourceRanges: []
 
 # Weaviate Config
 #