databricks · arpitjasa-db · Nov 8, 2023 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023
diff --git a/library/input_validation.tmpl b/library/input_validation.tmpl
@@ -33,11 +33,6 @@
         {{ fail `Invalid project name. Valid project names must contain at least three alphanumeric characters.` }}
     {{- end -}}
 
-    - Validate cicd platform
-    {{ if and (eq .input_cloud `aws`) (eq .input_cicd_platform `azure_devops`) -}}
-        {{ fail `Azure DevOps is not supported as a cicd_platform option with cloud=aws. If cloud=aws the currently supported cicd_platform is GitHub Actions.` }}
-    {{- end -}}
-
     - Validate schema_name for invalid characters
     {{- if eq .input_include_models_in_unity_catalog `yes` -}}
         {{- if ((regexp `[ ./\\]+`).MatchString .input_schema_name) -}}

diff --git a/.../{{.input_root_dir}}/.azure/devops-pipelines/{{.input_project_name}}-bundle-cicd.yml.tmpl b/.../{{.input_root_dir}}/.azure/devops-pipelines/{{.input_project_name}}-bundle-cicd.yml.tmpl
@@ -53,9 +53,14 @@ stages:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: 'Validate bundle for staging'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)             
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
+
 
   - job: prodBundleCI
     displayName: 'Prod bundle validation for {{template `project_name` .}}'
@@ -75,15 +80,19 @@ stages:
         curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sh   
       displayName: 'Install Databricks CLI'          
 
-    # Validate bundle to be deployed to the staging workspace
+    # Validate bundle to be deployed to the prod workspace
     - script: |
         databricks bundle validate -t prod
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
-      displayName: 'Validate bundle for staging'
+      displayName: 'Validate bundle for prod'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(PROD_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET) 
+        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
 # Run StagingBundleCD stage after successfully merging into the {{template `default_branch` .}} branch
 - stage: StagingBundleCD
@@ -122,19 +131,27 @@ stages:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: 'Validate bundle for staging'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET) 
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
     # Deploy bundle to Staging workspace
     - script: |
         databricks bundle deploy -t staging
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: 'Deploy bundle to staging'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)            
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
 # Run prod bundle CD stage after successfully merging into the {{template `release_branch` .}} branch
 - stage: prodBundleCD
@@ -173,16 +190,24 @@ stages:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: 'Validate bundle for prod'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(PROD_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET) 
+        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
     # Deploy bundle to prod workspace
     - script: |
         databricks bundle deploy -t prod
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: 'Deploy bundle to prod'
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(PROD_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(PROD_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(PROD_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)   
+        ARM_CLIENT_SECRET: $(PROD_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
diff --git a/...ate/{{.input_root_dir}}/.azure/devops-pipelines/{{.input_project_name}}-tests-ci.yml.tmpl b/...ate/{{.input_root_dir}}/.azure/devops-pipelines/{{.input_project_name}}-tests-ci.yml.tmpl
@@ -80,19 +80,27 @@ jobs:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: Validate bundle for test deployment target in staging workspace
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
         ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
     # Deploy bundle to staging workspace
     - script: |
         databricks bundle deploy -t test
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: Deploy bundle to test deployment target in staging workspace
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)          
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
 
     {{ if (eq .input_include_feature_store `yes`) }}
     # Run Feature Engineering Workflow for test deployment target in Staging Workspace
@@ -101,9 +109,13 @@ jobs:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: Run Feature Engineering Workflow for test deployment target in Staging Workspace
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)         
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
     {{ end }}
 
     # Run model_training_job defined in bundle in the staging workspace
@@ -112,6 +124,10 @@ jobs:
       workingDirectory: {{template `project_name_alphanumeric_underscore` .}}
       displayName: Run training workflow for test deployment target in staging workspace
       env:
+        {{ if (eq .input_cloud `aws`) -}}
+        DATABRICKS_TOKEN: $(STAGING_WORKSPACE_TOKEN)
+        {{- else if (eq .input_cloud `azure`) -}}
         ARM_TENANT_ID: $(STAGING_AZURE_SP_TENANT_ID)
         ARM_CLIENT_ID: $(STAGING_AZURE_SP_APPLICATION_ID)
-        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)          
+        ARM_CLIENT_SECRET: $(STAGING_AZURE_SP_CLIENT_SECRET)
+        {{- end }}
diff --git a/template/{{.input_root_dir}}/docs/mlops-setup.md.tmpl b/template/{{.input_root_dir}}/docs/mlops-setup.md.tmpl
@@ -144,12 +144,18 @@ By default, we provide Azure Pipelines where authentication is done using servic
 to get your service principal credentials (tenant id, application id, and client secret) for both the staging and prod service principals. You will use these credentials as variables in the project Azure Pipelines.
 1. Create two separate Azure Pipelines under your Azure DevOps project using the ‘Existing Azure Pipelines YAML file’ option. One of these pipelines will use the `{{template `project_name` .}}-tests-ci.yml` script, and the other will use the `{{template `project_name` .}}-bundles-cicd.yml` script. See [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/create-first-pipeline) for more details on creating Azure Pipelines.
 1. Create a new variable group called `{{template `project_name` .}} variable group` defining the following secret variables, for more details [here](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops&tabs=classic#create-a-variable-group):
+{{ if (eq .input_cloud `azure`) }}
     - `PROD_AZURE_SP_TENANT_ID`: tenant ID for the prod service principal
     - `PROD_AZURE_SP_APPLICATION_ID`: application (client) ID for the prod service principal
     - `PROD_AZURE_SP_CLIENT_SECRET`: client secret for the prod service principal
     - `STAGING_AZURE_SP_TENANT_ID`: tenant ID for the staging service principal
     - `STAGING_AZURE_SP_APPLICATION_ID`: application (client) ID for the staging service principal
     - `STAGING_AZURE_SP_CLIENT_SECRET`: client secret for the prod service principal
+  {{ end }}
+{{ if (eq .input_cloud `aws`) }}
+    - `STAGING_WORKSPACE_TOKEN` : service principal token for staging workspace
+    - `PROD_WORKSPACE_TOKEN` : service principal token for prod workspace
+  {{ end }}
       - Ensure that the two Azure Pipelines created in the prior step have access to these variables by selecting the name of the pipelines under the 'Pipeline permissions' tab of this variable group.
       - Alternatively you could store these secrets in an [Azure Key Vault](https://learn.microsoft.com/en-us/azure/devops/pipelines/release/key-vault-in-own-project?view=azure-devops&tabs=portal) and link those secrets as variables to be used in the Pipelines.
 1. Define two [build validation branch policies](https://learn.microsoft.com/en-us/azure/devops/repos/git/branch-policies?view=azure-devops&tabs=browser#build-validation) for the `{{template `default_branch` .}}` branch using the two Azure build pipelines created in step 1. This is required so that any PR changes to the `{{template `default_branch` .}}` must build successfully before PRs can complete.