Skip to content

Commit 8644f2a

Browse files
cwraucwrau
committed
feat: delete statefulsets if volumeClaimTemplates changes
Signed-off-by: Chris Werner Rau <[email protected]>
1 parent 89cadc5 commit 8644f2a

File tree

7 files changed

+180
-2
lines changed

7 files changed

+180
-2
lines changed

‎charts/tempo-distributed/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -610,8 +610,9 @@ The memcached default args are removed and should be provided manually. The sett
610610
| ingester.initContainers | list | `[]` | |
611611
| ingester.labels | object | `{}` | Labels for the ingester StatefulSet |
612612
| ingester.nodeSelector | object | `{}` | Node selector for ingester pods |
613-
| ingester.persistence | object | `{"annotations":{},"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
613+
| ingester.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
614614
| ingester.persistence.annotations | object | `{}` | Annotations for ingester's persist volume claim |
615+
| ingester.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size |
615616
| ingester.persistence.enabled | bool | `false` | Enable creating PVCs which is required when using boltdb-shipper |
616617
| ingester.persistence.inMemory | bool | `false` | use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart** |
617618
| ingester.persistence.labels | object | `{}` | Labels for ingester's persist volume claim |
@@ -766,8 +767,9 @@ The memcached default args are removed and should be provided manually. The sett
766767
| metricsGenerator.maxUnavailable | int | `1` | Pod Disruption Budget maxUnavailable |
767768
| metricsGenerator.minReadySeconds | int | `10` | Minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing/terminating |
768769
| metricsGenerator.nodeSelector | object | `{}` | Node selector for metrics-generator pods |
769-
| metricsGenerator.persistence | object | `{"annotations":{},"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
770+
| metricsGenerator.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
770771
| metricsGenerator.persistence.annotations | object | `{}` | Annotations for metrics generator PVCs |
772+
| metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size. This means that the StatefulSet will be deleted, recreated (with the same name) and rolled when a change to the PVC size is detected. That way the PVC can be resized without manual intervention. |
771773
| metricsGenerator.persistence.enabled | bool | `false` | Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir |
772774
| metricsGenerator.persistence.labels | object | `{}` | Labels for metrics generator PVCs |
773775
| metricsGenerator.persistence.storageClass | string | `nil` | Storage class to be used. If defined, storageClassName: <storageClass>. If set to "-", storageClassName: "", which disables dynamic provisioning. If empty or set to null, no storageClassName spec is set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). |

‎charts/tempo-distributed/templates/_helpers.tpl‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,3 +310,161 @@ Cluster name that shows up in dashboard metrics
310310
{{- define "tempo.clusterName" -}}
311311
{{ (include "tempo.calculatedConfig" . | fromYaml).cluster_name | default .Release.Name }}
312312
{{- end -}}
313+
314+
{{- define "tempo.statefulset.recreateOnSizeChangeHook" -}}
315+
{{- $renderedStatefulSets := list -}}
316+
{{- range $renderedStatefulSet := include (print .context.Template.BasePath .pathToStatefulsetTemplate) .context | splitList "---" -}}
317+
{{- with $renderedStatefulSet | fromYaml -}}
318+
{{- $renderedStatefulSets = append $renderedStatefulSets . -}}
319+
{{- end }}
320+
{{- end -}}
321+
{{- if $renderedStatefulSets }}
322+
{{- range $newStatefulSet := $renderedStatefulSets -}}
323+
{{- $currentStatefulset := dict -}}
324+
{{- if $newStatefulSet.spec.volumeClaimTemplates }}
325+
{{- $currentStatefulset = lookup $newStatefulSet.apiVersion $newStatefulSet.kind $newStatefulSet.metadata.namespace $newStatefulSet.metadata.name -}}
326+
{{- $needsRecreation := false -}}
327+
{{- $templates := dict -}}
328+
{{- if $currentStatefulset -}}
329+
{{- if ne (len $newStatefulSet.spec.volumeClaimTemplates) (len $currentStatefulset.spec.volumeClaimTemplates) -}}
330+
{{- $needsRecreation = true -}}
331+
{{- end -}}
332+
{{- range $index, $newVolumeClaimTemplate := $newStatefulSet.spec.volumeClaimTemplates -}}
333+
{{- $currentVolumeClaimTemplateSpec := dict -}}
334+
{{- range $oldVolumeClaimTemplate := $currentStatefulset.spec.volumeClaimTemplates -}}
335+
{{- if eq $oldVolumeClaimTemplate.metadata.name $newVolumeClaimTemplate.metadata.name -}}
336+
{{- $currentVolumeClaimTemplateSpec = $oldVolumeClaimTemplate.spec -}}
337+
{{- end -}}
338+
{{- end }}
339+
{{- $newVolumeClaimTemplateStorageSize := $newVolumeClaimTemplate.spec.resources.requests.storage -}}
340+
{{- if not $currentVolumeClaimTemplateSpec -}}
341+
{{- $needsRecreation = true -}}
342+
{{- else -}}
343+
{{- if ne $newVolumeClaimTemplateStorageSize $currentVolumeClaimTemplateSpec.resources.requests.storage -}}
344+
{{- $needsRecreation = true -}}
345+
{{- $templates = set $templates $newVolumeClaimTemplate.metadata.name $newVolumeClaimTemplateStorageSize -}}
346+
{{- end -}}
347+
{{- end -}}
348+
{{- end -}}
349+
{{- end -}}
350+
{{- if $needsRecreation }}
351+
apiVersion: batch/v1
352+
kind: Job
353+
metadata:
354+
name: {{ $newStatefulSet.metadata.name }}-recreate
355+
namespace: {{ $newStatefulSet.metadata.namespace }}
356+
labels:
357+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
358+
app.kubernetes.io/component: statefulset-recreate-job
359+
annotations:
360+
"helm.sh/hook": pre-upgrade
361+
"helm.sh/hook-weight": "-5"
362+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
363+
spec:
364+
ttlSecondsAfterFinished: 300
365+
template:
366+
metadata:
367+
name: {{ $newStatefulSet.metadata.name }}-recreate
368+
labels:
369+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 8 }}
370+
spec:
371+
serviceAccountName: {{ $newStatefulSet.metadata.name }}-recreate
372+
restartPolicy: OnFailure
373+
containers:
374+
- name: recreate
375+
image: {{ printf "%s/kubectl:%s" (coalesce $.context.Values.global.image.registry "registry.k8s.io") $.context.Capabilities.KubeVersion.Version }}
376+
command:
377+
- kubectl
378+
- delete
379+
- statefulset
380+
- {{ $newStatefulSet.metadata.name }}
381+
- --cascade=orphan
382+
{{- range $index := until (int $currentStatefulset.spec.replicas) }}
383+
{{- range $template, $size := $templates }}
384+
- name: patch-pvc-{{ $template }}-{{ $index }}
385+
image: {{ printf "%s/kubectl:%s" (coalesce $.context.Values.global.image.registry "registry.k8s.io") $.context.Capabilities.KubeVersion.Version }}
386+
command:
387+
- patch
388+
- pvc
389+
- --namespace={{ $newStatefulSet.metadata.namespace }}
390+
- {{ printf "%s-%s-%d" $template $newStatefulSet.metadata.name $index }}
391+
- --type='json'
392+
- '-p=[{"op": "replace", "path": "/spec/resources/requests/storage", "value": "{{ $size }}"}]'
393+
{{- end }}
394+
{{- end }}
395+
---
396+
apiVersion: v1
397+
kind: ServiceAccount
398+
metadata:
399+
name: {{ $newStatefulSet.metadata.name }}-recreate
400+
namespace: {{ $newStatefulSet.metadata.namespace }}
401+
labels:
402+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
403+
app.kubernetes.io/component: statefulset-recreate-job
404+
annotations:
405+
"helm.sh/hook": pre-upgrade
406+
"helm.sh/hook-weight": "-10"
407+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
408+
---
409+
apiVersion: rbac.authorization.k8s.io/v1
410+
kind: Role
411+
metadata:
412+
name: {{ $newStatefulSet.metadata.name }}-recreate
413+
namespace: {{ $newStatefulSet.metadata.namespace }}
414+
labels:
415+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
416+
app.kubernetes.io/component: statefulset-recreate-job
417+
annotations:
418+
"helm.sh/hook": pre-upgrade
419+
"helm.sh/hook-weight": "-10"
420+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
421+
rules:
422+
- apiGroups:
423+
- apps
424+
resources:
425+
- statefulsets
426+
resourceNames:
427+
- {{ $newStatefulSet.metadata.name }}
428+
verbs:
429+
- delete
430+
{{- if $templates }}
431+
- apiGroups:
432+
- v1
433+
resources:
434+
- persistentvolumeclaims
435+
resourceNames:
436+
{{- range $index := until (int $currentStatefulset.spec.replicas) }}
437+
{{- range $template := $templates | keys }}
438+
- {{ printf "%s-%s-%d" $template $newStatefulSet.metadata.name $index }}
439+
{{- end }}
440+
{{- end }}
441+
verbs:
442+
- patch
443+
{{- end }}
444+
---
445+
apiVersion: rbac.authorization.k8s.io/v1
446+
kind: RoleBinding
447+
metadata:
448+
name: {{ $newStatefulSet.metadata.name }}-recreate
449+
namespace: {{ $newStatefulSet.metadata.namespace }}
450+
labels:
451+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
452+
app.kubernetes.io/component: statefulset-recreate-job
453+
annotations:
454+
"helm.sh/hook": pre-upgrade
455+
"helm.sh/hook-weight": "-10"
456+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
457+
subjects:
458+
- kind: ServiceAccount
459+
name: {{ $newStatefulSet.metadata.name }}-recreate
460+
namespace: {{ $newStatefulSet.metadata.namespace }}
461+
roleRef:
462+
kind: Role
463+
name: {{ $newStatefulSet.metadata.name }}-recreate
464+
apiGroup: rbac.authorization.k8s.io
465+
---
466+
{{- end }}
467+
{{ end }}
468+
{{ end }}
469+
{{ end }}
470+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.ingester.persistence.enabled .Values.ingester.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/ingester/statefulset-ingester.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ spec:
4949
{{- end }}
5050
annotations:
5151
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
52+
{{- if .Values.ingester.persistence.enabled }}
53+
storage/size: {{ .Values.ingester.persistence.size | quote }}
54+
{{- end }}
5255
{{- with .Values.tempo.podAnnotations }}
5356
{{- toYaml . | nindent 8 }}
5457
{{- end }}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.metricsGenerator.persistence.enabled .Values.metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/metrics-generator/statefulset-metrics-generator.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ spec:
3636
{{- end }}
3737
annotations:
3838
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
39+
{{- if .Values.metricsGenerator.persistence.enabled }}
40+
storage/size: {{ .Values.metricsGenerator.persistence.size | quote }}
41+
{{- end }}
3942
{{- with .Values.tempo.podAnnotations }}
4043
{{- toYaml . | nindent 8 }}
4144
{{- end }}

‎charts/tempo-distributed/values.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,8 @@ ingester:
233233
persistence:
234234
# -- Enable creating PVCs which is required when using boltdb-shipper
235235
enabled: false
236+
# -- Enable StatefulSetRecreation for changes to PVC size
237+
enableStatefulSetRecreationForSizeChange: false
236238
# -- use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart**
237239
inMemory: false
238240
# -- Size of persistent or memory disk
@@ -432,6 +434,10 @@ metricsGenerator:
432434
persistence:
433435
# -- Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir
434436
enabled: false
437+
# -- Enable StatefulSetRecreation for changes to PVC size.
438+
# This means that the StatefulSet will be deleted, recreated (with the same name) and rolled when a change to the
439+
# PVC size is detected. That way the PVC can be resized without manual intervention.
440+
enableStatefulSetRecreationForSizeChange: false
435441
size: 10Gi
436442
# -- Storage class to be used.
437443
# If defined, storageClassName: <storageClass>.

0 commit comments

Comments
 (0)