Skip to content

Commit e87bd1e

Browse files
asdaraujoasdaraujo
authored
Adding support for SQL Stream Builder deployment (#48)
* New role to deploy the PostgreSQL connector Signed-off-by: Andre Araujo <[email protected]> * Added support for SQL Stream Builder Signed-off-by: Andre Araujo <[email protected]>
1 parent 321b94f commit e87bd1e

File tree

14 files changed

+184
-35
lines changed

14 files changed

+184
-35
lines changed

‎roles/config/cluster/base/templates/configs/databases-7.1.0.j2‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,18 @@ SCHEMAREGISTRY:
3131
database_name: {{ databases.SCHEMAREGISTRY.name }}
3232
database_user: {{ databases.SCHEMAREGISTRY.user }}
3333
database_password: {{ databases.SCHEMAREGISTRY.password }}
34+
SQL_STREAM_BUILDER:
35+
SERVICEWIDE:
36+
database_host: {{ databases.SQL_STREAM_BUILDER.host }}
37+
database_port: {{ databases.SQL_STREAM_BUILDER.port }}
38+
database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }}
39+
database_schema: {{ databases.SQL_STREAM_BUILDER.name }}
40+
database_user: {{ databases.SQL_STREAM_BUILDER.user }}
41+
database_password: {{ databases.SQL_STREAM_BUILDER.password }}
42+
MATERIALIZED_VIEW_ENGINE:
43+
ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }}
44+
ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }}
45+
ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }}
3446
STREAMS_MESSAGING_MANAGER:
3547
SERVICEWIDE:
3648
smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }}

‎roles/config/cluster/base/templates/configs/inter-service-dependencies.j2‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,11 @@ HUE:
7171
FLINK:
7272
SERVICEWIDE:
7373
hdfs_service: hdfs
74-
hive_service: hive
7574
yarn_service: yarn
76-
zookeeper_service: zoookeeper
75+
zookeeper_service: zookeeper
76+
{% if 'HIVE' in cluster.services %}
77+
hive_service: hive
78+
{% endif %}
7779
{% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
7880
atlas_service: atlas
7981
{% endif %}
@@ -174,6 +176,17 @@ SPARK3_ON_YARN:
174176
hive_service: hive
175177
{% endif %}
176178

179+
SQL_STREAM_BUILDER:
180+
SERVICEWIDE:
181+
flink_service: flink
182+
kafka_service: kafka
183+
{% if 'HIVE' in cluster.services %}
184+
hive_service: hive
185+
{% endif %}
186+
{% if 'KNOX' in cluster.services %}
187+
knox_service: knox
188+
{% endif %}
189+
177190
TEZ:
178191
SERVICEWIDE:
179192
yarn_service: yarn

‎roles/config/cluster/base/templates/configs/kerberos-6.x-7.x.j2‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
CORE_SETTINGS:
33
SERVICEWIDE:
44
hadoop_secure_web_ui: true
5+
FLINK:
6+
SERVICEWIDE:
7+
kerberos.auth.enabled: true
58
HBASE:
69
SERVICEWIDE:
710
hbase_restserver_security_authentication: kerberos

‎roles/config/cluster/base/templates/configs/tls-7.3.1.j2‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,26 @@
11
---
22
OZONE:
33
OZONE_PROMETHEUS:
4-
ozone.prometheus.ca.file: {{ tls_chain_path }}
4+
ozone.prometheus.ca.file: {{ tls_chain_path }}
5+
SQL_STREAM_BUILDER:
6+
STREAMING_SQL_ENGINE:
7+
ssl_client_truststore_location: {{ tls_truststore_path }}
8+
ssl_client_truststore_password: {{ tls_truststore_password }}
9+
ssl_enabled: true
10+
ssl_server_keystore_keypassword: {{ tls_keystore_password }}
11+
ssl_server_keystore_location: {{ tls_keystore_path_generic }}
12+
ssl_server_keystore_password: {{ tls_keystore_password }}
13+
MATERIALIZED_VIEW_ENGINE:
14+
ssl_client_truststore_location: {{ tls_truststore_path }}
15+
ssl_client_truststore_password: {{ tls_truststore_password }}
16+
ssl_enabled: true
17+
ssl_server_keystore_keypassword: {{ tls_keystore_password }}
18+
ssl_server_keystore_location: {{ tls_keystore_path_generic }}
19+
ssl_server_keystore_password: {{ tls_keystore_password }}
20+
STREAMING_SQL_CONSOLE:
21+
ssl_client_truststore_location: {{ tls_chain_path }}
22+
ssl_enabled: true
23+
ssl_server_ca_certificate_location: {{ tls_chain_path }}
24+
ssl_server_certificate_location: {{ tls_cert_path_generic }}
25+
ssl_server_privatekey_location: {{ tls_key_path_generic }}
26+
ssl_server_privatekey_password: {{ tls_key_password }}

‎roles/config/cluster/common/defaults/main.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ cluster_services_ordered:
8181
- CRUISE_CONTROL
8282
- DAS
8383
- FLINK
84+
- SQL_STREAM_BUILDER
8485
- SPARK
8586
- SPARK2_ON_YARN
8687
- SPARK3_ON_YARN

‎roles/deployment/databases/tasks/mariadb.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
encoding: "{{ service | cloudera.cluster.get_database_encoding_mysql }}"
2121
collation: "{{ service | cloudera.cluster.get_database_collation_mysql }}"
2222
become: yes
23-
loop: "{{ databases | intersect(services) }}"
23+
loop: "{{ databases }}"
2424
loop_control:
2525
loop_var: service
2626
delegate_to: "{{ databases[service].host }}"
@@ -36,7 +36,7 @@
3636
priv: "{{ databases[service].name }}.*:ALL"
3737
no_log: yes
3838
become: yes
39-
loop: "{{ databases | intersect(services) }}"
39+
loop: "{{ databases }}"
4040
loop_control:
4141
loop_var: service
4242
delegate_to: "{{ databases[service].host }}"

‎roles/deployment/databases/tasks/postgresql.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
password: "{{ databases[item].password }}"
2121
become: yes
2222
become_user: postgres
23-
with_items: "{{ databases | intersect(services) }}"
23+
with_items: "{{ databases }}"
2424
delegate_to: "{{ databases[item].host }}"
2525
connection: ssh
2626
when: databases[item].host in groups.db_server
@@ -32,7 +32,7 @@
3232
encoding: UTF-8
3333
become: yes
3434
become_user: postgres
35-
with_items: "{{ databases | intersect(services) }}"
35+
with_items: "{{ databases }}"
3636
delegate_to: "{{ databases[item].host }}"
3737
connection: ssh
3838
when: databases[item].host in groups.db_server

‎roles/deployment/definition/defaults/main.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,20 @@ database_defaults:
179179
name: schemaregistry
180180
user: schemaregistry
181181
password: "{{ database_default_password }}"
182+
SQL_STREAM_BUILDER:
183+
host: "{{ database_host }}"
184+
port: "{{ database_type | cloudera.cluster.default_database_port }}"
185+
type: "{{ database_type }}"
186+
name: ssb_admin
187+
user: ssb_admin
188+
password: "{{ database_default_password }}"
189+
SQL_STREAM_BUILDER_MVE:
190+
host: "{{ database_host }}"
191+
port: "{{ database_type | cloudera.cluster.default_database_port }}"
192+
type: "{{ database_type }}"
193+
name: ssb_mve
194+
user: ssb_mve
195+
password: "{{ database_default_password }}"
182196
STREAMS_MESSAGING_MANAGER:
183197
host: "{{ database_host }}"
184198
port: "{{ database_type | cloudera.cluster.default_database_port }}"

‎roles/deployment/repometa/templates/all_services.j2‎

Lines changed: 29 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,57 +1,58 @@
1-
- KEYTRUSTEE_SERVER
2-
- ZOOKEEPER
3-
- INFRA_SOLR
4-
- RANGER
5-
- RANGER_KMS
6-
- RANGER_KMS_KTS
7-
- RANGER_RAZ
8-
- KMS
9-
- HDFS
10-
- SENTRY
11-
- LIVY
12-
- HBASE
13-
- SOLR
14-
- SQOOP
15-
- SQOOP_CLIENT
161
- ACCUMULO16
172
- ACCUMULO_C6
18-
- HIVE
19-
- TEZ
20-
- HIVE_ON_TEZ
21-
- KUDU
22-
- IMPALA
23-
- YARN
243
- ADLS_CONNECTOR
4+
- ATLAS
255
- AWS_S3
266
- CDSW
277
- CORE_SETTINGS
288
- CRUISE_CONTROL
299
- DAS
3010
- FLINK
3111
- FLUME
32-
- SPARK
33-
- SPARK2_ON_YARN
34-
- SPARK3_ON_YARN
35-
- SPARK_ON_YARN
12+
- HBASE
13+
- HDFS
14+
- HIVE
15+
- HIVE_ON_TEZ
16+
- HUE
17+
- IMPALA
18+
- INFRA_SOLR
3619
- ISILON
3720
- KAFKA
3821
- KEYTRUSTEE
22+
- KEYTRUSTEE_SERVER
23+
- KMS
3924
- KNOX
4025
- KS_INDEXER
26+
- KUDU
27+
- LIVY
4128
- LUNA_KMS
4229
- MAPREDUCE
4330
- NIFI
4431
- NIFIREGISTRY
4532
- NIFITOOLKITCA
33+
- OOZIE
4634
- OZONE
4735
- PHOENIX
4836
- QUEUEMANAGER
37+
- RANGER
38+
- RANGER_KMS
39+
- RANGER_KMS_KTS
40+
- RANGER_RAZ
4941
- SCHEMAREGISTRY
42+
- SENTRY
43+
- SOLR
44+
- SPARK
45+
- SPARK2_ON_YARN
46+
- SPARK3_ON_YARN
47+
- SPARK_ON_YARN
48+
- SQL_STREAM_BUILDER
49+
- SQOOP
50+
- SQOOP_CLIENT
5051
- STREAMS_MESSAGING_MANAGER
5152
- STREAMS_REPLICATION_MANAGER
53+
- TEZ
5254
- THALES_KMS
5355
- WXM
56+
- YARN
5457
- ZEPPELIN
55-
- ATLAS
56-
- OOZIE
57-
- HUE
58+
- ZOOKEEPER

‎roles/deployment/repometa/templates/role_mappings/flink.j2‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
11
FLINK:
22
- FLINK_HISTORY_SERVER
33
- GATEWAY
4+
SQL_STREAM_BUILDER:
5+
- STREAMING_SQL_ENGINE
6+
- MATERIALIZED_VIEW_ENGINE
7+
- STREAMING_SQL_CONSOLE

0 commit comments

Comments
 (0)