fixes

Author: marcadamsge

cmds/core-service/main.go

@@ -112,7 +112,17 @@ func createAuxServer(ctx context.Context, locality string, publicEndpoint string
 		return nil, stacktrace.Propagate(err, "Unable to interact with store")
 	}
 
-	err = repo.SaveOwnMetadata(ctx, locality, publicEndpoint)
+	// SaveOwnMetadata is proposed at startup, before any HTTP request exists, so the
+	// context has neither a request timestamp nor a request seed (normally set by the
+	// HTTP middleware). Stamp both: the proposal carries them to all nodes so that
+	// every node derives the same time- and seed-dependent values when applying it.
+	seed, err := random.NewSeed()
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "failed to generate request seed")
+	}
+	metaCtx := random.NewContext(timestamp.WithRequestTimestamp(ctx, time.Now()), seed)
+
+	err = repo.SaveOwnMetadata(metaCtx, locality, publicEndpoint)
 
 	if err != nil {
 		return nil, stacktrace.Propagate(err, "Unable to store current metadata")

deploy/services/helm-charts/dss/templates/dss-raft.yaml

@@ -40,6 +40,11 @@ metadata:
     name: {{.Release.Name}}-core-service
   name: {{.Release.Name}}-core-service
 spec:
+  # All nodes must start together: each node blocks at startup on its first proposal
+  # (its own metadata), which needs a quorum, and the HTTP server only starts once the
+  # stores are initialized. With the default OrderedReady policy, pod-0 could never
+  # become ready before pods 1..N start, deadlocking the bootstrap.
+  podManagementPolicy: Parallel
   minReadySeconds: 30
   serviceName: {{.Release.Name}}-core-service-headless
   replicas: {{ $raft.replicas }}
@@ -68,38 +73,38 @@ spec:
               exec core-service \
                 --accepted_jwt_audiences={{$dss.conf.hostname}} \
                 --addr=:8080 \
-{{ if $dss.conf.publicEndpoint }}
+{{- if $dss.conf.publicEndpoint }}
                 --public_endpoint={{$dss.conf.publicEndpoint}} \
-{{ end }}
+{{- end }}
                 --locality={{$dss.conf.locality}} \
                 --store_type=raft \
                 --raft_node_id=$(( ${HOSTNAME##*-} + {{ $raft.nodeIDBase }} )) \
                 --raft_cluster_id={{ $raft.clusterID }} \
                 --raft_datadir={{ $raft.datadir }} \
-{{ range $flag, $value := $raft.tuning }}
+{{- range $flag, $value := $raft.tuning }}
                 --{{ $flag }}={{ $value }} \
-{{ end }}
+{{- end }}
                 --rid_raft_peers='{{ $ridPeers }}' \
                 --scd_raft_peers='{{ $scdPeers }}' \
                 --aux_raft_peers='{{ $auxPeers }}' \
                 --enable_scd={{$dss.conf.enableScd}} \
                 --enable_scd_global_lock={{$dss.conf.enableScdGlobalLock | default false}} \
-{{ if $dss.conf.enableTimeBasedNotificationIndex }}
+{{- if $dss.conf.enableTimeBasedNotificationIndex }}
                 --enable_time_based_notification_index=true \
-{{ end }}
-{{ if $dss.conf.enableDssMetrics }}
+{{- end }}
+{{- if $dss.conf.enableDssMetrics }}
                 --enable_metrics=true \
-{{ end }}
+{{- end }}
                 --gcp_prof_service_name= \
-{{ if $dss.conf.jwksEndpoint }}
+{{- if $dss.conf.jwksEndpoint }}
                 --jwks_endpoint={{ $dss.conf.jwksEndpoint }} \
-{{ end }}
-{{ if $dss.conf.jwksKeyIds }}
+{{- end }}
+{{- if $dss.conf.jwksKeyIds }}
                 --jwks_key_ids={{ $dss.conf.jwksKeyIds | join "," }} \
-{{ end }}
-{{ if $dss.conf.pubKeys }}
+{{- end }}
+{{- if $dss.conf.pubKeys }}
                 --public_key_files={{ $dss.conf.pubKeys | join "," }} \
-{{ end }}
+{{- end }}
           image: {{ $dss.image }}
           imagePullPolicy: {{ $dss.imagePullPolicy | default "Always" }}
           ports:
@@ -115,22 +120,31 @@ spec:
               name: raft-rid
             - containerPort: {{ $raft.ports.scd }}
               name: raft-scd
+          # TCP probe on the first Raft listener (aux): the transport is up as soon as
+          # the node is initialized, i.e. before it blocks on its first proposal
+          # (which needs a quorum). Readiness must not depend on quorum: headless
+          # services only publish DNS records for Ready pods, and peers are reached
+          # through exactly those records, so a /healthy-based readiness would
+          # deadlock the bootstrap (no quorum -> not ready -> no DNS -> no quorum).
           readinessProbe:
-            httpGet:
-              path: /healthy
-              port: 8080
+            tcpSocket:
+              port: raft-aux
           resources:
 {{ toYaml $raft.resources | indent 12 }}
           stdin: false
           tty: false
           volumeMounts:
             - mountPath: {{ $raft.datadir }}
               name: raft-data
+            {{- if $dss.conf.pubKeys }}
             {{- include "public-certs:volumeMount" (dict "cockroachdbEnabled" false) | nindent 12 }}
+            {{- end }}
       imagePullSecrets: []
       terminationGracePeriodSeconds: 30
+      {{- if $dss.conf.pubKeys }}
       volumes:
         {{- include "public-certs:volume" (dict "cockroachdbEnabled" false) | nindent 8 }}
+      {{- end }}
   volumeClaimTemplates:
     - metadata:
         name: raft-data

pkg/random/random.go

@@ -26,6 +26,12 @@ func newSeed() (int64, error) {
 	return int64(binary.BigEndian.Uint64(buf[:])), nil
 }
 
+// NewSeed returns a fresh random seed. The HTTP middleware stamps every request with one;
+// proposals submitted outside of a request (e.g. at startup) must be stamped by the caller.
+func NewSeed() (int64, error) {
+	return newSeed()
+}
+
 // MustFromContext returns the request seed from the context and panics if it is not present,
 // which is a programming error.
 func MustFromContext(ctx context.Context) int64 {