[raft] tanka deployment

deploy/services/tanka/core-service-args.libsonnet

@@ -0,0 +1,29 @@
+// Common core-service CLI parameters of the SQL and Raft deployments
+{
+  all(metadata): {
+    addr: ':' + std.toString(metadata.backend.port),
+    gcp_prof_service_name: metadata.backend.prof_grpc_name,
+    dump_requests: metadata.backend.dumpRequests,
+    accepted_jwt_audiences: metadata.backend.hostname,
+    locality: metadata.locality,
+    enable_scd: metadata.enableScd,
+    enable_scd_global_lock: metadata.enableScdGlobalLock,
+  }
+  + (if std.join(",", metadata.backend.pubKeys) != '' then {
+    public_key_files: std.join(",", metadata.backend.pubKeys),
+  } else {}) + (if metadata.backend.jwksEndpoint != '' then {
+    jwks_endpoint: metadata.backend.jwksEndpoint,
+  } else {}) + (if metadata.backend.jwksKeyIds != [] && metadata.backend.jwksKeyIds != [''] then {
+    jwks_key_ids: std.join(",", metadata.backend.jwksKeyIds),
+  } else {}) + (if metadata.backend.jwksRefreshInterval != '' then {
+    jwks_refresh_interval: metadata.backend.jwksRefreshInterval,
+  } else {}) + (if metadata.backend.jwksKeyTtl != '' then {
+    jwks_key_ttl: metadata.backend.jwksKeyTtl,
+  } else {}) + (if metadata.backend.publicEndpoint != '' then {
+    public_endpoint: metadata.backend.publicEndpoint,
+  } else {}) + (if metadata.enableDssMetrics then {
+    enable_metrics: true,
+  } else {}) + (if metadata.enableTimeBasedNotificationIndex then {
+    enable_time_based_notification_index: true,
+  } else {})
+}

deploy/services/tanka/core-service.libsonnet

@@ -1,102 +1,11 @@
 local base = import 'base.libsonnet';
 local util = import 'util.libsonnet';
 local volumes = import 'volumes.libsonnet';
+local args = import 'core-service-args.libsonnet';
 local datastoreparameters = import 'datastoreparameters.libsonnet';
 
-local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, 'gateway', [metadata.backend.ipName], metadata.subnet, metadata.backend.certName) {
-  app:: 'core-service',
-  spec+: {
-    ports: [{
-      port: 443,
-      targetPort: metadata.backend.port,
-      protocol: "TCP",
-      name: "http",
-    }]
-  }
-};
-
-local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') {
-  app:: 'core-service',
-  port:: metadata.backend.port,
-  type:: 'LoadBalancer',
-  spec+: {
-    ports: [{
-      port: 80,
-      targetPort: metadata.backend.port,
-      protocol: "TCP",
-      name: "http",
-    }]
-  }
-};
-
 {
-  GoogleIngress(metadata): base.Ingress(metadata, 'https-ingress') {
-    metadata+: {
-      annotations: {
-        'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName,
-        'kubernetes.io/ingress.allow-http': 'false',
-        [if metadata.backend.sslPolicy != '' then 'networking.gke.io/v1beta1.FrontendConfig']: 'ssl-frontend-config',
-      },
-    },
-    spec: {
-      defaultBackend: {
-        service: {
-          name: 'core-service',
-          port: {
-            number: metadata.backend.port,
-          }
-        }
-      },
-    },
-  },
-
-  GoogleSSLPolicyFrontendConfig(metadata): base.GoogleFrontendConfig(metadata, 'ssl-frontend-config', metadata.backend.sslPolicy),
-
-  GoogleManagedCertIngress(metadata): {
-    ingress: $.GoogleIngress(metadata) {
-      metadata+: {
-        annotations+: {
-          'networking.gke.io/managed-certificates': 'https-certificate',
-        },
-      },
-    },
-    managedCert: base.ManagedCert(metadata, 'https-certificate') {
-      spec: {
-        domains: [
-          metadata.backend.hostname,
-        ],
-      },
-    },
-  },
-
-  GooglePresharedCertIngress(metadata, certName): $.GoogleIngress(metadata) {
-    metadata+: {
-      annotations+: {
-        'ingress.gcp.kubernetes.io/pre-shared-cert': certName,
-      },
-    },
-  },
-
-  GoogleService(metadata): base.Service(metadata, 'core-service') {
-    app:: 'core-service',
-    port:: metadata.backend.port,
-    type:: 'NodePort',
-    enable_monitoring:: false,
-  },
-
-  CloudNetwork(metadata): {
-    google: if metadata.cloud_provider == "google" then {
-      ingress: $.GoogleManagedCertIngress(metadata),
-      [if metadata.backend.sslPolicy != '' then 'frontendConfig']: $.GoogleSSLPolicyFrontendConfig(metadata),
-      service: $.GoogleService(metadata),
-    },
-    aws_loadbalancer: if metadata.cloud_provider == "aws" then awsLoadBalancer(metadata),
-    minikube_loadbalancer: if metadata.cloud_provider == "minikube" then minikubeLoadBalancer(metadata),
-  },
-
   all(metadata): {
-    network: $.CloudNetwork(metadata),
-
     deployment: base.Deployment(metadata, 'core-service') {
       apiVersion: 'apps/v1',
       kind: 'Deployment',
@@ -136,32 +45,7 @@ local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') {
               env: util.makeEnv(metadata.backend.env),
               resources: util.resources(metadata.backend),
               command: ['core-service'],
-              args_:: {
-                addr: ':' + metadata.backend.port,
-                gcp_prof_service_name: metadata.backend.prof_grpc_name,
-                dump_requests: metadata.backend.dumpRequests,
-                accepted_jwt_audiences: metadata.backend.hostname,
-                locality: metadata.locality,
-                enable_scd: metadata.enableScd,
-                enable_scd_global_lock: metadata.enableScdGlobalLock,
-              } + datastoreparameters.all(metadata)
-              + (if std.join(",", metadata.backend.pubKeys) != '' then {
-                public_key_files: std.join(",", metadata.backend.pubKeys),
-              } else {}) + (if metadata.backend.jwksEndpoint != '' then {
-                jwks_endpoint: metadata.backend.jwksEndpoint,
-              } else {}) + (if metadata.backend.jwksKeyIds != [] && metadata.backend.jwksKeyIds != [''] then {
-                jwks_key_ids: std.join(",", metadata.backend.jwksKeyIds),
-              } else {}) + (if metadata.backend.jwksRefreshInterval != '' then {
-                jwks_refresh_interval: metadata.backend.jwksRefreshInterval,
-              } else {}) + (if metadata.backend.jwksKeyTtl != '' then {
-                jwks_key_ttl: metadata.backend.jwksKeyTtl,
-              } else {}) + (if metadata.backend.publicEndpoint != '' then {
-                public_endpoint: metadata.backend.publicEndpoint,
-              } else {}) + (if metadata.enableDssMetrics then {
-                enable_metrics: true,
-              } else {}) + (if metadata.enableTimeBasedNotificationIndex then {
-                enable_time_based_notification_index: true,
-              } else {}),
+              args_:: args.all(metadata) + datastoreparameters.all(metadata),
               readinessProbe: {
                 httpGet: {
                   path: '/healthy',

deploy/services/tanka/dss.libsonnet

@@ -3,6 +3,9 @@ local cockroachdb = import 'cockroachdb.libsonnet';
 local yugabyteAuxiliary = import 'yugabyte-auxiliary.libsonnet';
 local yugabyte = import 'yugabyte.libsonnet';
 local backend = import 'core-service.libsonnet';
+local raft = import 'raft.libsonnet';
+local raftAuxiliary = import 'raft-auxiliary.libsonnet';
+local ingress = import 'ingress.libsonnet';
 local base = import 'base.libsonnet';
 local prometheus = import 'prometheus.libsonnet';
 local grafana = import 'grafana.libsonnet';
@@ -41,11 +44,14 @@ local RoleBinding(metadata) = base.RoleBinding(metadata, 'default:privileged') {
     crdb_auxiliary: cockroachAuxiliary.all(metadata),
     yugabyte_sset: yugabyte.all(metadata),
     yugabyte_auxiliary: yugabyteAuxiliary.all(metadata),
-    backend: backend.all(metadata),
+    backend: if metadata.datastore == 'raft' then {} else backend.all(metadata),
+    raft: if metadata.datastore == 'raft' then raft.all(metadata) else {},
+    raft_auxiliary: if metadata.datastore == 'raft' then raftAuxiliary.all(metadata) else {},
+    ingress: ingress.all(metadata),
     prometheus: if metadata.enableMonitoring then prometheus.all(metadata),
     grafana: if metadata.enableMonitoring then grafana.all(metadata),
     alertmanager: if metadata.alert.enable == true then alertmanager.all(metadata),
-    schema_manager: if metadata.cockroach.shouldInit == true || metadata.schema_manager.enable then schema_manager.all(metadata) else {},
-    evict: evict.all(metadata),
+    schema_manager: if metadata.datastore != 'raft' && (metadata.cockroach.shouldInit == true || metadata.schema_manager.enable) then schema_manager.all(metadata) else {},
+    evict: if metadata.datastore != 'raft' then evict.all(metadata) else {},
   },
 }

deploy/services/tanka/ingress.libsonnet

@@ -0,0 +1,95 @@
+local base = import 'base.libsonnet';
+
+local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, 'gateway', [metadata.backend.ipName], metadata.subnet, metadata.backend.certName) {
+  app:: 'core-service',
+  spec+: {
+    ports: [{
+      port: 443,
+      targetPort: metadata.backend.port,
+      protocol: "TCP",
+      name: "http",
+    }]
+  }
+};
+
+local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') {
+  app:: 'core-service',
+  port:: metadata.backend.port,
+  type:: 'LoadBalancer',
+  spec+: {
+    ports: [{
+      port: 80,
+      targetPort: metadata.backend.port,
+      protocol: "TCP",
+      name: "http",
+    }]
+  }
+};
+
+{
+  GoogleIngress(metadata): base.Ingress(metadata, 'https-ingress') {
+    metadata+: {
+      annotations: {
+        'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName,
+        'kubernetes.io/ingress.allow-http': 'false',
+        [if metadata.backend.sslPolicy != '' then 'networking.gke.io/v1beta1.FrontendConfig']: 'ssl-frontend-config',
+      },
+    },
+    spec: {
+      defaultBackend: {
+        service: {
+          name: 'core-service',
+          port: {
+            number: metadata.backend.port,
+          }
+        }
+      },
+    },
+  },
+
+  GoogleSSLPolicyFrontendConfig(metadata): base.GoogleFrontendConfig(metadata, 'ssl-frontend-config', metadata.backend.sslPolicy),
+
+  GoogleManagedCertIngress(metadata): {
+    ingress: $.GoogleIngress(metadata) {
+      metadata+: {
+        annotations+: {
+          'networking.gke.io/managed-certificates': 'https-certificate',
+        },
+      },
+    },
+    managedCert: base.ManagedCert(metadata, 'https-certificate') {
+      spec: {
+        domains: [
+          metadata.backend.hostname,
+        ],
+      },
+    },
+  },
+
+  GooglePresharedCertIngress(metadata, certName): $.GoogleIngress(metadata) {
+    metadata+: {
+      annotations+: {
+        'ingress.gcp.kubernetes.io/pre-shared-cert': certName,
+      },
+    },
+  },
+
+  GoogleService(metadata): base.Service(metadata, 'core-service') {
+    app:: 'core-service',
+    port:: metadata.backend.port,
+    type:: 'NodePort',
+    enable_monitoring:: false,
+  },
+
+  CloudNetwork(metadata): {
+    google: if metadata.cloud_provider == "google" then {
+      ingress: $.GoogleManagedCertIngress(metadata),
+      [if metadata.backend.sslPolicy != '' then 'frontendConfig']: $.GoogleSSLPolicyFrontendConfig(metadata),
+      service: $.GoogleService(metadata),
+    },
+    aws_loadbalancer: if metadata.cloud_provider == "aws" then awsLoadBalancer(metadata),
+    minikube_loadbalancer: if metadata.cloud_provider == "minikube" then minikubeLoadBalancer(metadata),
+  },
+
+  all(metadata): $.CloudNetwork(metadata),
+}

deploy/services/tanka/metadata_base.libsonnet

@@ -58,6 +58,19 @@
       zone: error 'must specify placement zone',
     },
   },
+  raft: {
+    // The DSS nodes deployed by this release: {id, endpoint, ip}
+    nodes: error 'must supply the raft nodes of this instance',
+    // The other nodes of the pool, deployed by other releases: {id, endpoint}
+    peers: [],
+    clusterID: error 'must supply the raft cluster ID',
+    // Secret with the mutual TLS material of the Raft listeners
+    tlsSecret: error 'must supply the name of the raft mutual TLS secret',
+    storage: {
+      size: '100Gi',
+      storageClass: '',
+    },
+  },
   PSP: {
     roleRef: '',
     roleBinding: false,

deploy/services/tanka/raft-auxiliary.libsonnet

@@ -0,0 +1,53 @@
+local base = import 'base.libsonnet';
+
+local googleRaftLB(metadata, name, ip, port) = base.Service(metadata, name) {
+  port:: port,
+  app:: 'core-service',
+  spec+: {
+    type: 'LoadBalancer',
+    loadBalancerIP: ip,
+  },
+};
+
+local awsRaftLB(metadata, name, ip, port) = base.AWSLoadBalancer(metadata, name, [ip], metadata.subnet) {
+  port:: port,
+  app:: 'core-service',
+};
+
+local minikubeRaftLB(metadata, name, ip, port) = base.Service(metadata, name) {
+  port:: port,
+  app:: 'core-service',
+  spec+: {
+    type: 'LoadBalancer',
+  },
+};
+
+local raftNodeLB(metadata, i, node) =
+  local endpoint = std.split(node.endpoint, ':'),
+         name = 'raft-external-node-' + std.toString(i),
+         ip = node.ip,
+         port = std.parseInt(endpoint[1]);
+  (if metadata.cloud_provider == 'google' then googleRaftLB(metadata, name, ip, port)
+   else if metadata.cloud_provider == 'aws' then awsRaftLB(metadata, name, ip, port)
+   else if metadata.cloud_provider == 'minikube' then minikubeRaftLB(metadata, name, ip, port)) {
+    metadata+: {
+      annotations+: {
+        'service.alpha.kubernetes.io/tolerate-unready-endpoints': 'true',
+      },
+    },
+    spec+: {
+      selector: { 'statefulset.kubernetes.io/pod-name': 'core-service-' + std.toString(i) },
+      publishNotReadyAddresses: true,
+    },
+  };
+
+{
+  all(metadata):
+    local nodes = metadata.raft.nodes;
+    {
+      NodeGateways: {
+        ['raft-external-node-' + std.toString(i)]: raftNodeLB(metadata, i, nodes[i])
+        for i in std.range(0, std.length(nodes) - 1)
+      },
+    },
+}