diff --git a/deploy/services/tanka/core-service-args.libsonnet b/deploy/services/tanka/core-service-args.libsonnet new file mode 100644 index 000000000..443034ceb --- /dev/null +++ b/deploy/services/tanka/core-service-args.libsonnet @@ -0,0 +1,29 @@ +// Common core-service CLI parameters of the SQL and Raft deployments +{ + all(metadata): { + addr: ':' + std.toString(metadata.backend.port), + gcp_prof_service_name: metadata.backend.prof_grpc_name, + dump_requests: metadata.backend.dumpRequests, + accepted_jwt_audiences: metadata.backend.hostname, + locality: metadata.locality, + enable_scd: metadata.enableScd, + enable_scd_global_lock: metadata.enableScdGlobalLock, + } + + (if std.join(",", metadata.backend.pubKeys) != '' then { + public_key_files: std.join(",", metadata.backend.pubKeys), + } else {}) + (if metadata.backend.jwksEndpoint != '' then { + jwks_endpoint: metadata.backend.jwksEndpoint, + } else {}) + (if metadata.backend.jwksKeyIds != [] && metadata.backend.jwksKeyIds != [''] then { + jwks_key_ids: std.join(",", metadata.backend.jwksKeyIds), + } else {}) + (if metadata.backend.jwksRefreshInterval != '' then { + jwks_refresh_interval: metadata.backend.jwksRefreshInterval, + } else {}) + (if metadata.backend.jwksKeyTtl != '' then { + jwks_key_ttl: metadata.backend.jwksKeyTtl, + } else {}) + (if metadata.backend.publicEndpoint != '' then { + public_endpoint: metadata.backend.publicEndpoint, + } else {}) + (if metadata.enableDssMetrics then { + enable_metrics: true, + } else {}) + (if metadata.enableTimeBasedNotificationIndex then { + enable_time_based_notification_index: true, + } else {}) +} diff --git a/deploy/services/tanka/core-service.libsonnet b/deploy/services/tanka/core-service.libsonnet index c82aa034d..6899d312e 100644 --- a/deploy/services/tanka/core-service.libsonnet +++ b/deploy/services/tanka/core-service.libsonnet @@ -1,102 +1,11 @@ local base = import 'base.libsonnet'; local util = import 'util.libsonnet'; local volumes = import 'volumes.libsonnet'; +local args = import 'core-service-args.libsonnet'; local datastoreparameters = import 'datastoreparameters.libsonnet'; -local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, 'gateway', [metadata.backend.ipName], metadata.subnet, metadata.backend.certName) { - app:: 'core-service', - spec+: { - ports: [{ - port: 443, - targetPort: metadata.backend.port, - protocol: "TCP", - name: "http", - }] - } -}; - -local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') { - app:: 'core-service', - port:: metadata.backend.port, - type:: 'LoadBalancer', - spec+: { - ports: [{ - port: 80, - targetPort: metadata.backend.port, - protocol: "TCP", - name: "http", - }] - } -}; - { - GoogleIngress(metadata): base.Ingress(metadata, 'https-ingress') { - metadata+: { - annotations: { - 'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName, - 'kubernetes.io/ingress.allow-http': 'false', - [if metadata.backend.sslPolicy != '' then 'networking.gke.io/v1beta1.FrontendConfig']: 'ssl-frontend-config', - }, - }, - spec: { - defaultBackend: { - service: { - name: 'core-service', - port: { - number: metadata.backend.port, - } - } - }, - }, - }, - - GoogleSSLPolicyFrontendConfig(metadata): base.GoogleFrontendConfig(metadata, 'ssl-frontend-config', metadata.backend.sslPolicy), - - GoogleManagedCertIngress(metadata): { - ingress: $.GoogleIngress(metadata) { - metadata+: { - annotations+: { - 'networking.gke.io/managed-certificates': 'https-certificate', - }, - }, - }, - managedCert: base.ManagedCert(metadata, 'https-certificate') { - spec: { - domains: [ - metadata.backend.hostname, - ], - }, - }, - }, - - GooglePresharedCertIngress(metadata, certName): $.GoogleIngress(metadata) { - metadata+: { - annotations+: { - 'ingress.gcp.kubernetes.io/pre-shared-cert': certName, - }, - }, - }, - - GoogleService(metadata): base.Service(metadata, 'core-service') { - app:: 'core-service', - port:: metadata.backend.port, - type:: 'NodePort', - enable_monitoring:: false, - }, - - CloudNetwork(metadata): { - google: if metadata.cloud_provider == "google" then { - ingress: $.GoogleManagedCertIngress(metadata), - [if metadata.backend.sslPolicy != '' then 'frontendConfig']: $.GoogleSSLPolicyFrontendConfig(metadata), - service: $.GoogleService(metadata), - }, - aws_loadbalancer: if metadata.cloud_provider == "aws" then awsLoadBalancer(metadata), - minikube_loadbalancer: if metadata.cloud_provider == "minikube" then minikubeLoadBalancer(metadata), - }, - all(metadata): { - network: $.CloudNetwork(metadata), - deployment: base.Deployment(metadata, 'core-service') { apiVersion: 'apps/v1', kind: 'Deployment', @@ -136,32 +45,7 @@ local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') { env: util.makeEnv(metadata.backend.env), resources: util.resources(metadata.backend), command: ['core-service'], - args_:: { - addr: ':' + metadata.backend.port, - gcp_prof_service_name: metadata.backend.prof_grpc_name, - dump_requests: metadata.backend.dumpRequests, - accepted_jwt_audiences: metadata.backend.hostname, - locality: metadata.locality, - enable_scd: metadata.enableScd, - enable_scd_global_lock: metadata.enableScdGlobalLock, - } + datastoreparameters.all(metadata) - + (if std.join(",", metadata.backend.pubKeys) != '' then { - public_key_files: std.join(",", metadata.backend.pubKeys), - } else {}) + (if metadata.backend.jwksEndpoint != '' then { - jwks_endpoint: metadata.backend.jwksEndpoint, - } else {}) + (if metadata.backend.jwksKeyIds != [] && metadata.backend.jwksKeyIds != [''] then { - jwks_key_ids: std.join(",", metadata.backend.jwksKeyIds), - } else {}) + (if metadata.backend.jwksRefreshInterval != '' then { - jwks_refresh_interval: metadata.backend.jwksRefreshInterval, - } else {}) + (if metadata.backend.jwksKeyTtl != '' then { - jwks_key_ttl: metadata.backend.jwksKeyTtl, - } else {}) + (if metadata.backend.publicEndpoint != '' then { - public_endpoint: metadata.backend.publicEndpoint, - } else {}) + (if metadata.enableDssMetrics then { - enable_metrics: true, - } else {}) + (if metadata.enableTimeBasedNotificationIndex then { - enable_time_based_notification_index: true, - } else {}), + args_:: args.all(metadata) + datastoreparameters.all(metadata), readinessProbe: { httpGet: { path: '/healthy', diff --git a/deploy/services/tanka/dss.libsonnet b/deploy/services/tanka/dss.libsonnet index 28acc7c24..f93003887 100644 --- a/deploy/services/tanka/dss.libsonnet +++ b/deploy/services/tanka/dss.libsonnet @@ -3,6 +3,9 @@ local cockroachdb = import 'cockroachdb.libsonnet'; local yugabyteAuxiliary = import 'yugabyte-auxiliary.libsonnet'; local yugabyte = import 'yugabyte.libsonnet'; local backend = import 'core-service.libsonnet'; +local raft = import 'raft.libsonnet'; +local raftAuxiliary = import 'raft-auxiliary.libsonnet'; +local ingress = import 'ingress.libsonnet'; local base = import 'base.libsonnet'; local prometheus = import 'prometheus.libsonnet'; local grafana = import 'grafana.libsonnet'; @@ -41,11 +44,14 @@ local RoleBinding(metadata) = base.RoleBinding(metadata, 'default:privileged') { crdb_auxiliary: cockroachAuxiliary.all(metadata), yugabyte_sset: yugabyte.all(metadata), yugabyte_auxiliary: yugabyteAuxiliary.all(metadata), - backend: backend.all(metadata), + backend: if metadata.datastore == 'raft' then {} else backend.all(metadata), + raft: if metadata.datastore == 'raft' then raft.all(metadata) else {}, + raft_auxiliary: if metadata.datastore == 'raft' then raftAuxiliary.all(metadata) else {}, + ingress: ingress.all(metadata), prometheus: if metadata.enableMonitoring then prometheus.all(metadata), grafana: if metadata.enableMonitoring then grafana.all(metadata), alertmanager: if metadata.alert.enable == true then alertmanager.all(metadata), - schema_manager: if metadata.cockroach.shouldInit == true || metadata.schema_manager.enable then schema_manager.all(metadata) else {}, - evict: evict.all(metadata), + schema_manager: if metadata.datastore != 'raft' && (metadata.cockroach.shouldInit == true || metadata.schema_manager.enable) then schema_manager.all(metadata) else {}, + evict: if metadata.datastore != 'raft' then evict.all(metadata) else {}, }, } diff --git a/deploy/services/tanka/ingress.libsonnet b/deploy/services/tanka/ingress.libsonnet new file mode 100644 index 000000000..6f1640c04 --- /dev/null +++ b/deploy/services/tanka/ingress.libsonnet @@ -0,0 +1,95 @@ +local base = import 'base.libsonnet'; + +local awsLoadBalancer(metadata) = base.AWSLoadBalancerWithManagedCert(metadata, 'gateway', [metadata.backend.ipName], metadata.subnet, metadata.backend.certName) { + app:: 'core-service', + spec+: { + ports: [{ + port: 443, + targetPort: metadata.backend.port, + protocol: "TCP", + name: "http", + }] + } +}; + +local minikubeLoadBalancer(metadata) = base.Service(metadata, 'gateway') { + app:: 'core-service', + port:: metadata.backend.port, + type:: 'LoadBalancer', + spec+: { + ports: [{ + port: 80, + targetPort: metadata.backend.port, + protocol: "TCP", + name: "http", + }] + } +}; + +{ + GoogleIngress(metadata): base.Ingress(metadata, 'https-ingress') { + metadata+: { + annotations: { + 'kubernetes.io/ingress.global-static-ip-name': metadata.backend.ipName, + 'kubernetes.io/ingress.allow-http': 'false', + [if metadata.backend.sslPolicy != '' then 'networking.gke.io/v1beta1.FrontendConfig']: 'ssl-frontend-config', + }, + }, + spec: { + defaultBackend: { + service: { + name: 'core-service', + port: { + number: metadata.backend.port, + } + } + }, + }, + }, + + GoogleSSLPolicyFrontendConfig(metadata): base.GoogleFrontendConfig(metadata, 'ssl-frontend-config', metadata.backend.sslPolicy), + + GoogleManagedCertIngress(metadata): { + ingress: $.GoogleIngress(metadata) { + metadata+: { + annotations+: { + 'networking.gke.io/managed-certificates': 'https-certificate', + }, + }, + }, + managedCert: base.ManagedCert(metadata, 'https-certificate') { + spec: { + domains: [ + metadata.backend.hostname, + ], + }, + }, + }, + + GooglePresharedCertIngress(metadata, certName): $.GoogleIngress(metadata) { + metadata+: { + annotations+: { + 'ingress.gcp.kubernetes.io/pre-shared-cert': certName, + }, + }, + }, + + GoogleService(metadata): base.Service(metadata, 'core-service') { + app:: 'core-service', + port:: metadata.backend.port, + type:: 'NodePort', + enable_monitoring:: false, + }, + + CloudNetwork(metadata): { + google: if metadata.cloud_provider == "google" then { + ingress: $.GoogleManagedCertIngress(metadata), + [if metadata.backend.sslPolicy != '' then 'frontendConfig']: $.GoogleSSLPolicyFrontendConfig(metadata), + service: $.GoogleService(metadata), + }, + aws_loadbalancer: if metadata.cloud_provider == "aws" then awsLoadBalancer(metadata), + minikube_loadbalancer: if metadata.cloud_provider == "minikube" then minikubeLoadBalancer(metadata), + }, + + all(metadata): $.CloudNetwork(metadata), +} diff --git a/deploy/services/tanka/metadata_base.libsonnet b/deploy/services/tanka/metadata_base.libsonnet index d44e88f27..e68ffff39 100644 --- a/deploy/services/tanka/metadata_base.libsonnet +++ b/deploy/services/tanka/metadata_base.libsonnet @@ -58,6 +58,19 @@ zone: error 'must specify placement zone', }, }, + raft: { + // The DSS nodes deployed by this release: {id, endpoint, ip} + nodes: error 'must supply the raft nodes of this instance', + // The other nodes of the pool, deployed by other releases: {id, endpoint} + peers: [], + clusterID: error 'must supply the raft cluster ID', + // Secret with the mutual TLS material of the Raft listeners + tlsSecret: error 'must supply the name of the raft mutual TLS secret', + storage: { + size: '100Gi', + storageClass: '', + }, + }, PSP: { roleRef: '', roleBinding: false, diff --git a/deploy/services/tanka/raft-auxiliary.libsonnet b/deploy/services/tanka/raft-auxiliary.libsonnet new file mode 100644 index 000000000..0f15c40a2 --- /dev/null +++ b/deploy/services/tanka/raft-auxiliary.libsonnet @@ -0,0 +1,53 @@ +local base = import 'base.libsonnet'; + +local googleRaftLB(metadata, name, ip, port) = base.Service(metadata, name) { + port:: port, + app:: 'core-service', + spec+: { + type: 'LoadBalancer', + loadBalancerIP: ip, + }, +}; + +local awsRaftLB(metadata, name, ip, port) = base.AWSLoadBalancer(metadata, name, [ip], metadata.subnet) { + port:: port, + app:: 'core-service', +}; + +local minikubeRaftLB(metadata, name, ip, port) = base.Service(metadata, name) { + port:: port, + app:: 'core-service', + spec+: { + type: 'LoadBalancer', + }, +}; + +local raftNodeLB(metadata, i, node) = + local endpoint = std.split(node.endpoint, ':'), + name = 'raft-external-node-' + std.toString(i), + ip = node.ip, + port = std.parseInt(endpoint[1]); + (if metadata.cloud_provider == 'google' then googleRaftLB(metadata, name, ip, port) + else if metadata.cloud_provider == 'aws' then awsRaftLB(metadata, name, ip, port) + else if metadata.cloud_provider == 'minikube' then minikubeRaftLB(metadata, name, ip, port)) { + metadata+: { + annotations+: { + 'service.alpha.kubernetes.io/tolerate-unready-endpoints': 'true', + }, + }, + spec+: { + selector: { 'statefulset.kubernetes.io/pod-name': 'core-service-' + std.toString(i) }, + publishNotReadyAddresses: true, + }, + }; + +{ + all(metadata): + local nodes = metadata.raft.nodes; + { + NodeGateways: { + ['raft-external-node-' + std.toString(i)]: raftNodeLB(metadata, i, nodes[i]) + for i in std.range(0, std.length(nodes) - 1) + }, + }, +} diff --git a/deploy/services/tanka/raft.libsonnet b/deploy/services/tanka/raft.libsonnet new file mode 100644 index 000000000..e1ca610b0 --- /dev/null +++ b/deploy/services/tanka/raft.libsonnet @@ -0,0 +1,105 @@ +local base = import 'base.libsonnet'; +local util = import 'util.libsonnet'; +local volumes = import 'volumes.libsonnet'; +local args = import 'core-service-args.libsonnet'; + +{ + all(metadata): { + local raft = metadata.raft, + local nodes = raft.nodes, + local allNodes = nodes + raft.peers, + local ids = [n.id for n in allNodes], + local badEndpoints = [n.endpoint for n in allNodes + if std.length(std.split(n.endpoint, '://')) > 1 || std.length(std.split(n.endpoint, ':')) != 2], + local duplicateIds = [i for i in std.set(ids) if std.count(ids, i) > 1], + + // Flat values indexed by pod ordinal + local idsList = std.join(' ', [std.toString(n.id) for n in nodes]), + // The --raft_peers list is nodes + peers (self included), all addressed over https. + local peersList = std.join(',', [std.toString(n.id) + '=https://' + n.endpoint for n in allNodes]), + local ports = std.set([std.parseInt(std.split(n.endpoint, ':')[1]) for n in nodes]), + + local allArgs = args.all(metadata) + { + store_type: 'raft', + raft_node_id: '$(printf \'%s\\n\' $ids | sed -n "${ordinal}p")', + raft_cluster_id: raft.clusterID, + raft_datadir: '/raft_data', + raft_peers: peersList, + raft_tls_cert_file: '/raft-tls/tls.crt', + raft_tls_key_file: '/raft-tls/tls.key', + raft_tls_ca_file: '/raft-tls/ca-bundle.crt', + }, + local script = std.join('\n', [ + '# Raft node IDs of this instance\'s nodes, in pod-ordinal order.', + 'ids="' + idsList + '"', + 'ordinal=$(( ${HOSTNAME##*-} + 1 ))', + 'exec core-service ' + std.join(' \\\n ', + std.map(function(kv) + if kv[0] == 'raft_node_id' then '--raft_node_id=' + kv[1] + else '\'' + ('--%s=%s' % kv) + '\'', + util.objectItems(allArgs))), + ]), + + assert std.length(nodes) > 0 : 'raft.nodes must list at least one node', + assert badEndpoints == [] : 'raft node endpoints must be host:port without a scheme: ' + std.join(', ', badEndpoints), + assert duplicateIds == [] : 'duplicate Raft node IDs in raft.nodes / raft.peers: ' + std.join(', ', [std.toString(i) for i in duplicateIds]), + assert raft.tlsSecret != '' : 'raft.tlsSecret is required: the Raft listeners authenticate with mutual TLS', + + statefulset: base.StatefulSet(metadata, 'core-service') { + spec+: { + serviceName: null, + podManagementPolicy: 'Parallel', + minReadySeconds: 30, + replicas: std.length(nodes), + volumeClaimTemplates: [{ + metadata: { name: 'raft-data' }, + spec: { + accessModes: ['ReadWriteOnce'], + storageClassName: if raft.storage.storageClass != '' then raft.storage.storageClass else null, + resources: { requests: { storage: raft.storage.size } }, + }, + }], + template+: { + metadata+: { + annotations: if metadata.enableDssMetrics then { + 'prometheus.io/path': 'metrics', + 'prometheus.io/port': '8079', + 'prometheus.io/scrape': 'true', + } else {}, + }, + spec+: { + affinity: { + podAntiAffinity: { + preferredDuringSchedulingIgnoredDuringExecution: [{ + weight: 100, + podAffinityTerm: { + labelSelector: { matchLabels: { app: 'core-service' } }, + topologyKey: 'kubernetes.io/hostname', + }, + }], + }, + }, + volumes: volumes.all(metadata).backendVolumes, + soloContainer:: base.Container('core-service') { + image: metadata.backend.image, + imagePullPolicy: if metadata.backend.imagePullPolicy != '' then metadata.backend.imagePullPolicy + else if metadata.cloud_provider == 'minikube' then 'IfNotPresent' else 'Always', + command: ['sh', '-c'], + args: [script], + ports: [ + { containerPort: metadata.backend.port, name: 'http' }, + ] + (if metadata.enableDssMetrics then [{ containerPort: 8079, name: 'metrics' }] else []) + + [{ containerPort: p } for p in ports], + volumeMounts: volumes.all(metadata).backendMounts, + env: util.makeEnv(metadata.backend.env), + resources: util.resources(metadata.backend), + readinessProbe: { + httpGet: { path: '/healthy', port: metadata.backend.port }, + }, + }, + }, + }, + }, + }, + }, +} diff --git a/deploy/services/tanka/volumes.libsonnet b/deploy/services/tanka/volumes.libsonnet index 2674b101b..3ed4402d3 100644 --- a/deploy/services/tanka/volumes.libsonnet +++ b/deploy/services/tanka/volumes.libsonnet @@ -79,6 +79,44 @@ local util = import 'util.libsonnet'; cockroachMounts: std.flattenArrays(util.mapToList(util.exclude(self.mounts, std.set(['publicCert'])))), backendMounts: std.flattenArrays(util.mapToList(util.exclude(self.mounts, std.set(['datadir', 'nodeCert'])))), schemaMounts: std.flattenArrays(util.mapToList(util.exclude(self.mounts, std.set(['datadir', 'nodeCert', 'publicCert'])))), + } else if metadata.datastore == "raft" then { + // The raft-data volume is not listed: the StatefulSet volumeClaimTemplate + // auto-injects it into the pod spec. + volumes: { + raft_tls: { + name: 'raft-tls', + secret: { + secretName: metadata.raft.tlsSecret, + defaultMode: 256, + }, + }, + public_certs: { + name: 'public-certs', + secret: { + secretName: 'dss.public.certs', + defaultMode: 256, + }, + }, + }, + backendVolumes: util.mapToList(self.volumes), + schemaVolumes: [], + + mounts: { + datadir: [{ + name: 'raft-data', + mountPath: '/raft_data', + }], + tls: [{ + name: 'raft-tls', + mountPath: '/raft-tls', + }], + publicCert: [{ + name: 'public-certs', + mountPath: '/public-certs', + }], + }, + backendMounts: std.flattenArrays(util.mapToList(self.mounts)), + schemaMounts: [], } else { volumes: { ca_certs: {