Skip to content

Commit a39e3fb

Browse files
marcadamsgemarcadamsge
committed
raft helm chart
1 parent a4cccdd commit a39e3fb

9 files changed

Lines changed: 327 additions & 4 deletions

File tree

‎deploy/services/helm-charts/dss/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Helm Chart
2-
This directory provides an [Helm Chart](https://helm.sh/) to deploy the DSS and datastore (CockroachDB or Yugabyte) kubernetes resources.
2+
This directory provides an [Helm Chart](https://helm.sh/) to deploy the DSS and datastore (CockroachDB, Yugabyte, or the Raft-based store) kubernetes resources. Exactly one datastore backend must be enabled: `cockroachdb.enabled`, `yugabyte.enabled` or `raft.enabled`.
33

44
## Requirements
55
1. A Kubernetes cluster should be running and you should be properly authenticated. Requirements and instructions to create a new Kubernetes cluster can be found here:

‎deploy/services/helm-charts/dss/templates/_helpers.tpl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,25 @@
1+
{{- define "dss.validate-datastore" -}}
2+
{{- $enabled := 0 -}}
3+
{{- if $.Values.cockroachdb.enabled -}}{{- $enabled = add1 $enabled -}}{{- end -}}
4+
{{- if $.Values.yugabyte.enabled -}}{{- $enabled = add1 $enabled -}}{{- end -}}
5+
{{- if $.Values.raft.enabled -}}{{- $enabled = add1 $enabled -}}{{- end -}}
6+
{{- if ne $enabled 1 -}}
7+
{{- fail (printf "Exactly one datastore backend must be enabled: set exactly one of cockroachdb.enabled, yugabyte.enabled and raft.enabled to true (got %d enabled)" $enabled) -}}
8+
{{- end -}}
9+
{{- end -}}
10+
11+
{{- define "dss.raft-peers" -}}
12+
{{- $releaseName := .releaseName -}}
13+
{{- $base := int .base -}}
14+
{{- $replicas := int .replicas -}}
15+
{{- $port := int .port -}}
16+
{{- $entries := list -}}
17+
{{- range $i := until $replicas -}}
18+
{{- $entries = append $entries (printf "%d=http://%s-core-service-%d.%s-core-service-headless:%d" (add $i $base) $releaseName $i $releaseName $port) -}}
19+
{{- end -}}
20+
{{- join "," $entries -}}
21+
{{- end -}}
22+
123
{{- define "datastoreImage" -}}
224
{{- if $.Values.cockroachdb.enabled -}}
325
{{ (printf "%s:%s" $.Values.cockroachdb.image.repository $.Values.cockroachdb.image.tag) }}

‎deploy/services/helm-charts/dss/templates/dss-core-service.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{{- if not .Values.raft.enabled }}
12
{{- $dss := .Values.dss -}}
23

34
{{- $datastoreImage := (include "datastoreImage" .) -}}
@@ -118,3 +119,4 @@ spec:
118119
{{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }}
119120
{{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }}
120121
{{- include "public-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }}
122+
{{- end }}

‎deploy/services/helm-charts/dss/templates/dss-evict.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
{{/* Eviction on the Raft store is not implemented yet. */}}
2+
{{- if not .Values.raft.enabled }}
13
{{- $image := .Values.dss.image }}
24

35
{{- $datastoreImage := (include "datastoreImage" .) -}}
@@ -134,3 +136,4 @@ spec:
134136
volumes:
135137
{{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }}
136138
{{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 12 }}
139+
{{- end }}

‎deploy/services/helm-charts/dss/templates/dss-raft.yaml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
{{/* Exactly one datastore backend must be enabled */}}
2+
{{- include "dss.validate-datastore" . -}}
3+
{{- if .Values.raft.enabled }}
4+
{{- $dss := .Values.dss -}}
5+
{{- $raft := .Values.raft -}}
6+
{{- $ridPeers := include "dss.raft-peers" (dict "releaseName" .Release.Name "base" $raft.nodeIDBase "replicas" $raft.replicas "port" $raft.ports.rid) -}}
7+
{{- $scdPeers := include "dss.raft-peers" (dict "releaseName" .Release.Name "base" $raft.nodeIDBase "replicas" $raft.replicas "port" $raft.ports.scd) -}}
8+
{{- $auxPeers := include "dss.raft-peers" (dict "releaseName" .Release.Name "base" $raft.nodeIDBase "replicas" $raft.replicas "port" $raft.ports.aux) -}}
9+
10+
---
11+
apiVersion: v1
12+
kind: Service
13+
metadata:
14+
labels:
15+
name: {{.Release.Name}}-core-service-headless
16+
name: {{.Release.Name}}-core-service-headless
17+
spec:
18+
clusterIP: None
19+
selector:
20+
app: {{.Release.Name}}-core-service
21+
ports:
22+
- port: {{ $raft.ports.aux }}
23+
targetPort: {{ $raft.ports.aux }}
24+
protocol: TCP
25+
name: raft-aux
26+
- port: {{ $raft.ports.rid }}
27+
targetPort: {{ $raft.ports.rid }}
28+
protocol: TCP
29+
name: raft-rid
30+
- port: {{ $raft.ports.scd }}
31+
targetPort: {{ $raft.ports.scd }}
32+
protocol: TCP
33+
name: raft-scd
34+
35+
---
36+
apiVersion: apps/v1
37+
kind: StatefulSet
38+
metadata:
39+
labels:
40+
name: {{.Release.Name}}-core-service
41+
name: {{.Release.Name}}-core-service
42+
spec:
43+
minReadySeconds: 30
44+
serviceName: {{.Release.Name}}-core-service-headless
45+
replicas: {{ $raft.replicas }}
46+
selector:
47+
matchLabels:
48+
app: {{.Release.Name}}-core-service
49+
template:
50+
metadata:
51+
labels:
52+
app: {{.Release.Name}}-core-service
53+
{{ if $dss.conf.enableDssMetrics }}
54+
annotations:
55+
prometheus.io/path: metrics
56+
prometheus.io/port: '8079'
57+
prometheus.io/scrape: 'true'
58+
{{ end }}
59+
spec:
60+
containers:
61+
- name: core-service
62+
{{/* The node ID is derived from the pod ordinal at runtime: pod <name>-N is raft node <N + nodeIDBase>. */}}
63+
command:
64+
- sh
65+
- -c
66+
args:
67+
- |
68+
exec core-service \
69+
--accepted_jwt_audiences={{$dss.conf.hostname}} \
70+
--addr=:8080 \
71+
{{ if $dss.conf.publicEndpoint }}
72+
--public_endpoint={{$dss.conf.publicEndpoint}} \
73+
{{ end }}
74+
--locality={{$dss.conf.locality}} \
75+
--store_type=raft \
76+
--raft_node_id=$(( ${HOSTNAME##*-} + {{ $raft.nodeIDBase }} )) \
77+
--raft_cluster_id={{ $raft.clusterID }} \
78+
--raft_datadir={{ $raft.datadir }} \
79+
{{ range $flag, $value := $raft.tuning }}
80+
--{{ $flag }}={{ $value }} \
81+
{{ end }}
82+
--rid_raft_peers='{{ $ridPeers }}' \
83+
--scd_raft_peers='{{ $scdPeers }}' \
84+
--aux_raft_peers='{{ $auxPeers }}' \
85+
--enable_scd={{$dss.conf.enableScd}} \
86+
--enable_scd_global_lock={{$dss.conf.enableScdGlobalLock | default false}} \
87+
{{ if $dss.conf.enableTimeBasedNotificationIndex }}
88+
--enable_time_based_notification_index=true \
89+
{{ end }}
90+
{{ if $dss.conf.enableDssMetrics }}
91+
--enable_metrics=true \
92+
{{ end }}
93+
--gcp_prof_service_name= \
94+
{{ if $dss.conf.jwksEndpoint }}
95+
--jwks_endpoint={{ $dss.conf.jwksEndpoint }} \
96+
{{ end }}
97+
{{ if $dss.conf.jwksKeyIds }}
98+
--jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} \
99+
{{ end }}
100+
{{ if $dss.conf.pubKeys }}
101+
--public_key_files={{ $dss.conf.pubKeys | join "," }} \
102+
{{ end }}
103+
image: {{ $dss.image }}
104+
imagePullPolicy: {{ $dss.imagePullPolicy | default "Always" }}
105+
ports:
106+
- containerPort: 8080
107+
name: http
108+
{{ if $dss.conf.enableDssMetrics }}
109+
- containerPort: 8079
110+
name: metrics
111+
{{ end }}
112+
- containerPort: {{ $raft.ports.aux }}
113+
name: raft-aux
114+
- containerPort: {{ $raft.ports.rid }}
115+
name: raft-rid
116+
- containerPort: {{ $raft.ports.scd }}
117+
name: raft-scd
118+
readinessProbe:
119+
httpGet:
120+
path: /healthy
121+
port: 8080
122+
resources:
123+
{{ toYaml $raft.resources | indent 12 }}
124+
stdin: false
125+
tty: false
126+
volumeMounts:
127+
- mountPath: {{ $raft.datadir }}
128+
name: raft-data
129+
{{- include "public-certs:volumeMount" (dict "cockroachdbEnabled" false) | nindent 12 }}
130+
imagePullSecrets: []
131+
terminationGracePeriodSeconds: 30
132+
volumes:
133+
{{- include "public-certs:volume" (dict "cockroachdbEnabled" false) | nindent 8 }}
134+
volumeClaimTemplates:
135+
- metadata:
136+
name: raft-data
137+
spec:
138+
accessModes:
139+
- ReadWriteOnce
140+
{{ if $raft.storage.storageClass }}
141+
storageClassName: {{ $raft.storage.storageClass }}
142+
{{ end }}
143+
resources:
144+
requests:
145+
storage: {{ $raft.storage.size }}
146+
{{- end }}

‎deploy/services/helm-charts/dss/templates/schema-manager.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
{{- $schemas = dict "rid" "1.0.1" "scd" "1.1.1" "aux_" "1.1.0" }}
1313
{{- end -}}
1414

15+
{{- if not .Values.raft.enabled }}
1516
{{- range $service, $schemaVersion := $schemas }}
1617
{{- $serviceClean := $service | replace "_" "" }}
1718
---
@@ -66,4 +67,5 @@ spec:
6667
volumes:
6768
{{- include "ca-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }}
6869
{{- include "client-certs:volume" (dict "cockroachdbEnabled" $.Values.cockroachdb.enabled ) | nindent 8 }}
69-
{{- end -}}
70+
{{- end }}
71+
{{- end }}

‎deploy/services/helm-charts/dss/values.example.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,32 @@ yugabyte:
6161
# If you join a yugabyte cluster, set the masterAddresses there
6262
# masterAddresses: "yb-master-0.yb-masters.default.svc.cluster.local:7100,yb-master-1.yb-masters.default.svc.cluster.local:7100,yb-master-2.yb-masters.default.svc.cluster.local:7100"
6363

64+
# To deploy with the Raft-based datastore instead of CockroachDB/Yugabyte, set
65+
# raft.enabled to true and disable both cockroachdb.enabled and yugabyte.enabled.
66+
# Exactly one datastore backend must be enabled. The DSS nodes are then deployed
67+
# as a StatefulSet with one PVC per node (Raft WAL and snapshots).
68+
raft:
69+
enabled: false
70+
replicas: 3 # Number of DSS nodes (Raft voters). Must be odd for a useful quorum (2n+1).
71+
# nodeIDBase: 1 # Raft node ID of pod 0 (pod N is nodeIDBase + N). Never change after bootstrap.
72+
# clusterID: 1 # Same value on every node of the pool; different from other reachable pools.
73+
# datadir: /raft_data
74+
# ports:
75+
# aux: 9031
76+
# rid: 9011
77+
# scd: 9021
78+
# storage:
79+
# size: 100Gi
80+
# storageClass: standard # Low fsync latency recommended (commit path writes the WAL).
81+
# resources:
82+
# requests:
83+
# cpu: 1
84+
# memory: 4Gi
85+
# limits:
86+
# cpu: 4
87+
# memory: 16Gi
88+
# tuning: {} # e.g. { "raft_tick_interval": "200ms" }
89+
6490
loadBalancers:
6591
cockroachdbNodes:
6692
- ip: 192.168.0.21

‎deploy/services/helm-charts/dss/values.schema.json‎

Lines changed: 86 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"properties": {
88
"enabled": {
99
"type": "boolean",
10-
"description": "Enable CockroachDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)."
10+
"description": "Enable CockroachDB. Exactly one of CockroachDB, YugabyteDB and Raft should be enabled (only one at a time)."
1111
},
1212
"image": {
1313
"type": "object",
@@ -83,7 +83,7 @@
8383
"properties": {
8484
"enabled": {
8585
"type": "boolean",
86-
"description": "Enable YugabyteDB. YugabyteDB xor CockroachDB should be enabled (only one at at time, at least one)."
86+
"description": "Enable YugabyteDB. Exactly one of CockroachDB, YugabyteDB and Raft should be enabled (only one at a time)."
8787
},
8888
"gflags": {
8989
"type": "object",
@@ -151,6 +151,90 @@
151151
],
152152
"additionalItems": true
153153
},
154+
"raft": {
155+
"description": "Raft-based datastore related configuration",
156+
"type": "object",
157+
"properties": {
158+
"enabled": {
159+
"type": "boolean",
160+
"description": "Enable the Raft-based datastore. Exactly one of raft, cockroachdb or yugabyte should be enabled (only one at a time)."
161+
},
162+
"replicas": {
163+
"type": "integer",
164+
"minimum": 1,
165+
"description": "Number of DSS nodes (Raft voters) deployed by this instance. An odd number (2n+1) is recommended so the pool tolerates the loss of n nodes."
166+
},
167+
"nodeIDBase": {
168+
"type": "integer",
169+
"minimum": 1,
170+
"description": "Raft node ID of the pod with ordinal 0. The node ID of pod ordinal N is nodeIDBase + N. Node IDs are never reused: do not change this value once the pool has been bootstrapped."
171+
},
172+
"clusterID": {
173+
"type": "integer",
174+
"minimum": 1,
175+
"description": "Raft cluster ID. Must be identical on all nodes of the pool, and different from any other pool whose nodes could reach each other over the network (it rejects cross-pool messages by mistake)."
176+
},
177+
"datadir": {
178+
"type": "string",
179+
"description": "Directory mounted on the PVC where the node persists its Raft data (WAL segments and snapshots). These must not be deleted across restarts unless the node is permanently removed from the pool."
180+
},
181+
"ports": {
182+
"type": "object",
183+
"properties": {
184+
"aux": {
185+
"type": "integer",
186+
"minimum": 1,
187+
"maximum": 65535,
188+
"description": "Raft listener port of the AUX service"
189+
},
190+
"rid": {
191+
"type": "integer",
192+
"minimum": 1,
193+
"maximum": 65535,
194+
"description": "Raft listener port of the RID service"
195+
},
196+
"scd": {
197+
"type": "integer",
198+
"minimum": 1,
199+
"maximum": 65535,
200+
"description": "Raft listener port of the SCD service"
201+
}
202+
},
203+
"required": [
204+
"aux",
205+
"rid",
206+
"scd"
207+
]
208+
},
209+
"storage": {
210+
"type": "object",
211+
"properties": {
212+
"size": {
213+
"type": "string",
214+
"description": "PVC size for the Raft data (WAL and snapshots). A low-fsync-latency volume is recommended: the WAL is written before a proposal is acknowledged."
215+
},
216+
"storageClass": {
217+
"type": "string",
218+
"description": "Storage class depending on your cloud provider. Empty for the cluster default."
219+
}
220+
},
221+
"required": [
222+
"size"
223+
]
224+
},
225+
"resources": {
226+
"type": "object",
227+
"description": "Standard Kubernetes resources block for the core service pods. The design document recommends about 16Gi of memory and 4 vCPU per node."
228+
},
229+
"tuning": {
230+
"type": "object",
231+
"description": "Additional raft_* flags passed to the core service, keyed by flag name without the leading --, e.g. { \"raft_tick_interval\": \"200ms\" }."
232+
}
233+
},
234+
"required": [
235+
"enabled"
236+
]
237+
},
154238
"loadBalancers": {
155239
"description": "Load balancers configuration",
156240
"type": "object",

0 commit comments

Comments
 (0)