Skip to content

Commit ebab278

Browse files
ravi-databricksravi-databricks
authored
Merge pull request #205 from databrickslabs/Issue_22
Added:
2 parents 4a11742 + 0158969 commit ebab278

File tree

4 files changed

+182
-1
lines changed

4 files changed

+182
-1
lines changed

‎demo/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -329,7 +329,7 @@ This demo showcases how to use Databricks Asset Bundles (DABs) with DLT-Meta:
329329
* * Implementing SCD Type 1 to Silver tables
330330
* * Applying expectations to filter data in Silver tables
331331
332-
## Prerequisites
332+
### Steps:
333333
1. Launch Command Prompt
334334
335335
2. Install [Databricks CLI](https://docs.databricks.com/dev-tools/cli/index.html)

‎examples/dab_onboarding_job.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Bronze and Silver DLT Meta Onboarding for People Data
2+
# yaml-language-server: $schema=../bundle_config_schema.json
3+
4+
resources:
5+
jobs:
6+
onboard_people:
7+
name: People - Onboard Job
8+
description: "Job to onboard DLT META data"
9+
10+
tasks:
11+
- task_key: onboard_bronze_silver_tables
12+
python_wheel_task:
13+
package_name: dlt_meta
14+
entry_point: run
15+
named_parameters:
16+
database: ${var.${bundle.target}_catalog_name}.${var.dlt_meta_schema}
17+
onboarding_file_path: ${var.people_onboarding_file_path}
18+
bronze_dataflowspec_table: ${var.bronze_dataflowspecTable}
19+
silver_dataflowspec_table: ${var.silver_dataflowspecTable}
20+
uc_enabled: ${var.uc_enabled}
21+
import_author: ${var.author}
22+
version: ${var.version}
23+
onboard_layer: bronze_silver
24+
overwrite: "True"
25+
env: ${bundle.target}
26+
27+
# job_cluster_key: my_job_cluster
28+
environment_key: default
29+
30+
- task_key: onboard_silver_fanout_tables
31+
python_wheel_task:
32+
package_name: dlt_meta
33+
entry_point: run
34+
named_parameters:
35+
database: ${var.${bundle.target}_catalog_name}.${var.dlt_meta_schema}
36+
onboarding_file_path: ${var.people_fanout_onboarding_file_path}
37+
silver_dataflowspec_table: ${var.silver_dataflowspecTable}
38+
uc_enabled: ${var.uc_enabled}
39+
import_author: ${var.author}
40+
version: ${var.version}
41+
onboard_layer: silver
42+
overwrite: "False"
43+
env: ${bundle.target}
44+
depends_on:
45+
- task_key: onboard_bronze_silver_tables
46+
47+
environment_key: default
48+
49+
environments:
50+
- environment_key: default
51+
spec:
52+
client: "1"
53+
dependencies:
54+
- dlt-meta
55+
56+
execute_pipelines_people:
57+
name: People - Raw_Br_Si Job
58+
description: "Job to execute the Bronze and Silver DLT-META pipelines"
59+
60+
tasks:
61+
- task_key: execute_bronze_people
62+
description: "Run DLT pipeline for Bronze People"
63+
pipeline_task:
64+
pipeline_id: ${resources.pipelines.pipeline_bronze_people.id}
65+
full_refresh: true
66+
67+
- task_key: execute_silver_people
68+
description: "Run DLT pipeline for Silver People"
69+
pipeline_task:
70+
pipeline_id: ${resources.pipelines.pipeline_silver_people.id}
71+
full_refresh: true
72+
depends_on:
73+
- task_key: execute_bronze_people

‎examples/dab_pipelines.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Bronze and Silver DLT Meta Pipelines for People Data
2+
resources:
3+
pipelines:
4+
pipeline_bronze_people:
5+
name: People - R2B Pipeline1
6+
catalog: ${var.${bundle.target}_catalog_name}
7+
schema: ${var.bronze_schema}
8+
development: ${var.${bundle.target}_development_enabled}
9+
photon: ${var.photon_enabled}
10+
serverless: ${var.serverless_enabled}
11+
libraries:
12+
- notebook:
13+
path: ${workspace.file_path}/notebooks/init_dlt_meta_pipeline
14+
15+
configuration:
16+
layer: bronze
17+
dummy_param: ${var.dummy_param}
18+
bronze.dataflowspecTable: ${var.${bundle.target}_catalog_name}.${var.dlt_meta_schema}.${var.bronze_dataflowspecTable}
19+
bronze.group: my_people
20+
bundle.sourcePath: ${workspace.file_path}/notebooks
21+
22+
pipeline_silver_people:
23+
name: People - B2S Pipeline1
24+
catalog: ${var.${bundle.target}_catalog_name}
25+
schema: ${var.silver_schema}
26+
development: ${var.${bundle.target}_development_enabled}
27+
photon: ${var.photon_enabled}
28+
serverless: ${var.serverless_enabled}
29+
libraries:
30+
- notebook:
31+
path: ${workspace.file_path}/notebooks/init_dlt_meta_pipeline
32+
33+
configuration:
34+
layer: silver
35+
dummy_param: ""
36+
silver.dataflowspecTable: ${var.${bundle.target}_catalog_name}.${var.dlt_meta_schema}.${var.silver_dataflowspecTable}
37+
silver.group: my_people
38+
bundle.sourcePath: ${workspace.file_path}/notebooks
39+

‎examples/var_dab_people.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
variables:
2+
dev_catalog_name:
3+
description: The catalog name for development environment
4+
type: string
5+
default: ravi_dlt_meta_uc
6+
prod_catalog_name:
7+
description: The catalog name for production environment
8+
type: string
9+
default: ravi_dlt_meta_uc
10+
dlt_meta_schema:
11+
description: The schema name for the pipelines
12+
type: string
13+
default: dlt_meta_dataflowspecs_demo_6d5fe507bd764d978bf89db089a570d8
14+
bronze_schema:
15+
description: The schema name for the bronze pipelines
16+
type: string
17+
default: dlt_meta_bronze_demo_6d5fe507bd764d978bf89db089a570d8
18+
silver_schema:
19+
description: The schema name for the silver pipelines
20+
type: string
21+
default: dlt_meta_silver_demo_6d5fe507bd764d978bf89db089a570d8
22+
photon_enabled:
23+
description: Whether Photon is enabled for the pipelines
24+
type: bool
25+
default: true
26+
serverless_enabled:
27+
description: Whether serverless mode is enabled for the pipelines
28+
type: bool
29+
default: true
30+
bronze_dataflowspecTable:
31+
description: The table name for the bronze data flow specification
32+
type: string
33+
default: bronze_dataflowspec_table
34+
silver_dataflowspecTable:
35+
description: The table name for the silver data flow specification
36+
type: string
37+
default: silver_dataflowspec_table
38+
author:
39+
description: The author of the import
40+
type: string
41+
default: [email protected]
42+
people_onboarding_file_path:
43+
description: The path to the onboarding file for people
44+
type: string
45+
default: /Volumes/ravi_dlt_meta_uc/dlt_meta_dataflowspecs_demo_6d5fe507bd764d978bf89db089a570d8/dlt_meta_files/demo/dabs/conf/onboarding_bronze_silver_people.json
46+
people_fanout_onboarding_file_path:
47+
description: The path to the onboarding file for people
48+
type: string
49+
default: /Volumes/ravi_dlt_meta_uc/dlt_meta_dataflowspecs_demo_6d5fe507bd764d978bf89db089a570d8/dlt_meta_files/demo/dabs/conf/onboarding_silver_fanout_people.json
50+
dummy_param:
51+
description: A dummy parameter for testing purposes
52+
type: string
53+
default: Hello Bronze 2
54+
version:
55+
description: The version of the data flow specification
56+
type: string
57+
default: v1
58+
uc_enabled:
59+
description: Whether Unity Catalog is enabled for the pipelines
60+
type: bool
61+
default: true
62+
dev_development_enabled:
63+
description: Whether development mode is enabled for the pipelines
64+
type: bool
65+
default: true
66+
prod_development_enabled:
67+
description: Whether production development mode is enabled for the pipelines
68+
type: bool
69+
default: false

0 commit comments

Comments
 (0)