Skip to content

Commit 96a922e

Browse files
ravi-databricksravi-databricks
committed
Adding generate_dabs_resource script for demo
1 parent 256484a commit 96a922e

File tree

1 file changed

+211
-0
lines changed

1 file changed

+211
-0
lines changed

‎demo/generate_dabs_resources.py‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
2+
import uuid
3+
import json
4+
import yaml
5+
from src.install import WorkspaceInstaller
6+
from integration_tests.run_integration_tests import (
7+
DLTMETARunner,
8+
DLTMetaRunnerConf,
9+
get_workspace_api_client,
10+
process_arguments
11+
)
12+
import traceback
13+
14+
15+
class DLTMETADABDemo(DLTMETARunner):
16+
17+
def __init__(self, args, ws, base_dir):
18+
self.args = args
19+
self.ws = ws
20+
self.wsi = WorkspaceInstaller(ws)
21+
self.base_dir = base_dir
22+
23+
def run(self, runner_conf: DLTMetaRunnerConf):
24+
"""
25+
Runs the DLT-META DAB Demo by calling the necessary methods in the correct order.
26+
27+
Parameters:
28+
- runner_conf: The DLTMetaRunnerConf object containing the runner configuration parameters.
29+
"""
30+
try:
31+
self.initialize_uc_resources(runner_conf)
32+
self.generate_var_people_yml(runner_conf) # Generate var_people.yml
33+
self.generate_onboarding_files(runner_conf)
34+
self.upload_files_to_databricks(runner_conf)
35+
except Exception as e:
36+
print(e)
37+
traceback.print_exc()
38+
# finally:
39+
# self.clean_up(runner_conf)
40+
41+
def init_runner_conf(self) -> DLTMetaRunnerConf:
42+
"""
43+
Initialize the runner configuration for running integration tests.
44+
45+
Returns:
46+
-------
47+
DLTMetaRunnerConf
48+
The initialized runner configuration.
49+
"""
50+
run_id = uuid.uuid4().hex
51+
runner_conf = DLTMetaRunnerConf(
52+
run_id=run_id,
53+
username=self.wsi._my_username,
54+
uc_catalog_name=self.args["uc_catalog_name"],
55+
int_tests_dir="demo/dabs",
56+
dlt_meta_schema=f"dlt_meta_dataflowspecs_demo_{run_id}",
57+
bronze_schema=f"dlt_meta_bronze_demo_{run_id}",
58+
silver_schema=f"dlt_meta_silver_demo_{run_id}",
59+
runners_nb_path=f"/Users/{self.wsi._my_username}/dlt_meta_demo/{run_id}",
60+
source="cloudfiles",
61+
env="demo",
62+
cloudfiles_template="demo/dabs/conf/onboarding_bronze_silver_people.template",
63+
onboarding_file_path="demo/dabs/conf/onboarding_bronze_silver_people.json",
64+
onboarding_fanout_templates="demo/dabs/conf/onboarding_silver_fanout_people.template",
65+
onboarding_fanout_file_path="demo/dabs/conf/onboarding_silver_fanout_people.json",
66+
runners_full_local_path='./demo/dabs/notebooks/',
67+
test_output_file_path=(
68+
f"/Users/{self.wsi._my_username}/dlt_meta_demo/"
69+
f"{run_id}/demo-output.csv"
70+
),
71+
)
72+
73+
return runner_conf
74+
75+
def generate_var_people_yml(self, runner_conf: DLTMetaRunnerConf):
76+
"""Generate var_people.yml with configuration from runner_conf."""
77+
uc_vol_full_path = f"{runner_conf.uc_volume_path}{runner_conf.int_tests_dir}"
78+
var_people_content = {
79+
"variables": {
80+
"dev_catalog_name": {
81+
"description": "The catalog name for development environment",
82+
"type": "string",
83+
"default": runner_conf.uc_catalog_name
84+
},
85+
"prod_catalog_name": {
86+
"description": "The catalog name for production environment",
87+
"type": "string",
88+
"default": runner_conf.uc_catalog_name
89+
},
90+
"dlt_meta_schema": {
91+
"description": "The schema name for the pipelines",
92+
"type": "string",
93+
"default": f"{runner_conf.dlt_meta_schema}"
94+
},
95+
"bronze_schema": {
96+
"description": "The schema name for the bronze pipelines",
97+
"type": "string",
98+
"default": f"{runner_conf.bronze_schema}"
99+
},
100+
"silver_schema": {
101+
"description": "The schema name for the silver pipelines",
102+
"type": "string",
103+
"default": f"{runner_conf.silver_schema}"
104+
},
105+
"photon_enabled": {
106+
"description": "Whether Photon is enabled for the pipelines",
107+
"type": "bool",
108+
"default": True
109+
},
110+
"serverless_enabled": {
111+
"description": "Whether serverless mode is enabled for the pipelines",
112+
"type": "bool",
113+
"default": True
114+
},
115+
"bronze_dataflowspecTable": {
116+
"description": "The table name for the bronze data flow specification",
117+
"type": "string",
118+
"default": "bronze_dataflowspec_table"
119+
},
120+
"silver_dataflowspecTable": {
121+
"description": "The table name for the silver data flow specification",
122+
"type": "string",
123+
"default": "silver_dataflowspec_table"
124+
},
125+
"author": {
126+
"description": "The author of the import",
127+
"type": "string",
128+
"default": runner_conf.username
129+
},
130+
"people_onboarding_file_path": {
131+
"description": "The path to the onboarding file for people",
132+
"type": "string",
133+
"default": f"{uc_vol_full_path}/conf/onboarding_bronze_silver_people.json"
134+
},
135+
"people_fanout_onboarding_file_path": {
136+
"description": "The path to the onboarding file for people",
137+
"type": "string",
138+
"default": f"{uc_vol_full_path}/conf/onboarding_silver_fanout_people.json"
139+
},
140+
"dummy_param": {
141+
"description": "A dummy parameter for testing purposes",
142+
"type": "string",
143+
"default": "Hello Bronze 2"
144+
},
145+
"version": {
146+
"description": "The version of the data flow specification",
147+
"type": "string",
148+
"default": "v1"
149+
},
150+
"uc_enabled": {
151+
"description": "Whether Unity Catalog is enabled for the pipelines",
152+
"type": "bool",
153+
"default": True
154+
},
155+
"dev_development_enabled": {
156+
"description": "Whether development mode is enabled for the pipelines",
157+
"type": "bool",
158+
"default": True
159+
},
160+
"prod_development_enabled": {
161+
"description": "Whether production development mode is enabled for the pipelines",
162+
"type": "bool",
163+
"default": False
164+
}
165+
}
166+
}
167+
var_people_path = f"{runner_conf.int_tests_dir}/resources/var_people.yml"
168+
with open(var_people_path, "w") as f:
169+
yaml.dump(var_people_content, f, sort_keys=False, default_flow_style=False)
170+
print(f"Generated var_people.yml at {var_people_path}")
171+
172+
def generate_onboarding_files(self, runner_conf: DLTMetaRunnerConf):
173+
string_subs = {
174+
"{uc_volume_path}": runner_conf.uc_volume_path,
175+
"{uc_catalog_name}": runner_conf.uc_catalog_name,
176+
"{bronze_schema}": runner_conf.bronze_schema,
177+
"{silver_schema}": runner_conf.silver_schema,
178+
}
179+
if runner_conf.cloudfiles_template:
180+
with open(f"{runner_conf.cloudfiles_template}", "r") as f:
181+
onboard_json = f.read()
182+
183+
for key, val in string_subs.items():
184+
val = "" if val is None else val # Ensure val is a string
185+
onboard_json = onboard_json.replace(key, val)
186+
with open(runner_conf.onboarding_file_path, "w") as onboarding_file:
187+
json.dump(json.loads(onboard_json), onboarding_file, indent=4)
188+
189+
if runner_conf.onboarding_fanout_templates:
190+
template = runner_conf.onboarding_fanout_templates
191+
with open(f"{template}", "r") as f:
192+
onboard_json = f.read()
193+
194+
for key, val in string_subs.items():
195+
onboard_json = onboard_json.replace(key, val)
196+
197+
with open(runner_conf.onboarding_fanout_file_path, "w") as onboarding_file:
198+
json.dump(json.loads(onboard_json), onboarding_file, indent=4)
199+
200+
201+
def main():
202+
args = process_arguments()
203+
workspace_client = get_workspace_api_client(args["profile"])
204+
dltmeta_afam_demo_runner = DLTMETADABDemo(args, workspace_client, "demo")
205+
print("initializing complete")
206+
runner_conf = dltmeta_afam_demo_runner.init_runner_conf()
207+
dltmeta_afam_demo_runner.run(runner_conf)
208+
209+
210+
if __name__ == "__main__":
211+
main()

0 commit comments

Comments
 (0)